o jl@sddlZddlZddlmmZddlmZddlmZddl m Z ddl Z ddl m Z dJddZdJdd Zd d Zd d ZddZdKddZddZddZddZddZddZddZddZdJd d!ZdKd"d#Zd$d%ZdJd&d'ZdLd*d+ZdMd-d.Z dJd/d0Z!dNd3d4Z"dKd5d6Z#dJd7d8Z$dJd9d:Z%d;d<Z&dJd=d>Z'd?e(d@e(dAe e e(fdBdCZ)dDdEZ*dFdGZ+dHdIZ,dS)ON)glob)Path)List)tqdmc tj||}t|ddd }|}Wdn1swYt|dd}t|ddD]\}}t|d|krNtd|dd | q3t j tj||dd t fd d d DshJdj vrodnd} dj vrxdnd} g} d} D]=} | dur|dur| j|vrqtj|| j}tj|s| d7} q| | j|| dur| n| j| dur| n| j|dq| dkrtd| d| S)zbNormalizes the CML-TTS meta data file to TTS format https://github.com/freds0/CML-TTS-Dataset/rutf8encodingNr| > Missing column in line  -> sepc3|]}|jvVqdSNcolumns.0xmetadataN/home/kuhnn/.local/lib/python3.10/site-packages/TTS/tts/datasets/formatters.py zcml_tts..) wav_filename transcript client_iddefault emotion_nameneutraltext audio_file speaker_namer" root_path | > [!]  files not found)ospathjoinopen readlineslensplit enumerateprintstrippdread_csvallr itertuplesr rexistsappendrr")r( meta_fileignored_speakersfilepathflinesnum_colsidxliner r"itemsnot_found_counterrow audio_pathrrrcml_ttssB    rGc r)zInteral dataset formatter.rrr Nrr r r rrc3rrrrrrrrCrzcoqui..)r&r%r'coquir"r#r$r)r*)r+r,r-r.r/r0r1r2r3r4r5r6r7rr8r'r&r9r:r%r")r(r;r<r=r>r?r@rArBr'r"rCrDrErFrrrrH7sB    rHc Kstj||}g}d}t|ddd,}|D] }|d}tj||dd} |d} || | ||d qWd |S1sAwY|S) zhNormalize TWEB dataset. https://www.kaggle.com/bryanpark/the-world-english-bible-speech-dataset twebrutf-8r  r.wavr r%r&r'r(Nr+r,r-r.r1r: r(r;kwargstxt_filerCr'ttfrBcolswav_filer%rrrrI]s  rIc Kstj||}g}d}t|ddd1}|D]%}|d}|d} |d} tj|d| } || | ||d qWd |S1sFwY|S) 0Normalizes Mozilla meta data files to TTS formatmozillarrJr r r rwavsrMN)r+r,r-r.r1r4r:rOrrrrVms    rVc Kstj||}g}d}t|ddd>}|D]2}|d}|d} |d} d| d dd } tj|| | } || | ||d qWd |S1sSwY|S) rUrVrz ISO 8859-1r r rr BATCH___FINALrMN)r+r,r-r.r4r1r:) r(r;rPrQrCr'rRrBrSrTr% folder_namerrr mozilla_de|s    r\c Cstdtjdtjdtjdtj}|s(t|tjdtjddd}n|}g}|D]}tj|r9|}ntj||}tj|}| |} | d urPq.| d } t |t r_| |vr_q.t d |t|d d dO} | D]D} | d} |stj|d| dd}ntj||ddd| dd}tj|r| d}|||| |dqpt d|qpWd n1swYq.|S)aNormalizes M-AI-Labs meta data files to TTS format Args: root_path (str): root folder of the MAILAB language folder. meta_files (str): list of meta files to be used in the training. If None, finds all the csv files recursively. Defaults to None by_bookz (male|female)z(?P[^z]+)z**z metadata.csvT recursiveNr'z | > {}rrJr r rWrrLr rMz> File %s does not exist!)recompiler+rrr,isfiler-dirnamesearchgroup isinstancelistr3formatr.r1replacer4r:)r( meta_filesr< speaker_regex csv_filesrCcsv_filerQfolderspeaker_name_matchr'rRrBrSrTr%rrrmailabssD*"      "   rqc Ktj||}g}d}t|ddd-}|D]!}|d}tj|d|dd} |d } || | ||d qWd |S1sBwY|S) z`Normalizes the LJSpeech meta data file to TTS format https://keithito.com/LJ-Speech-Dataset/ljspeechrrJr r rWrrLrMNrNrOrrrrs  rsc Kstj||}g}t|ddd@}d}t|D]0\}}|ddkr%|d7}|d} tj|d| dd } | d} || | d ||d qWd |S1sSwY|S) zbNormalizes the LJSpeech meta data file for TTS testing https://keithito.com/LJ-Speech-Dataset/rrJr rrtr r rWrLz ljspeech-rMN)r+r,r-r.r2r1r:) r(r;rPrQrCrR speaker_idrArBrSrTr%rrr ljspeech_tests$     rwc Krr) zuNormalizes the thorsten meta data file to TTS format https://github.com/thorstenMueller/deep-learning-german-tts/thorstenrrJr r rWrrLr rMNrNrOrrrrxrurxc Kstj|d|}t|}g}d}|dD]+}|j}tj|d|dd} tj | s9t d| dq| || ||d q|S) zNormalizes the sam-accenture meta data file to TTS format https://github.com/Sam-Accenture-Non-Binary-Voice/non-binary-voice-filesvoice_over_recordings sam_accenturez./fileidvo_voice_quality_transformationidrL [!] z( in metafile does not exist. Skipping...rM) r+r,r-ETparsegetrootfindallr%getr9r3r:) r(r;rPxml_filexml_rootrCr'itemr%rTrrrrzs rzc Krr) zWNormalizes the RUSLAN meta data file to TTS format https://ruslan-corpus.github.io/ruslanrrJr r RUSLANrrLr rMNrNrOrrrrrurc Ktj||}g}d}t|ddd*}|D]}|d}tj||d} |d} || | ||dqWd |S1s?wY|S) z/Normalizes the CSS10 dataset file to TTS formatcss10rrJr r rr rMNrNrOrrrr  rc Kstj||}g}d}t|ddd8}|D],}|d}||dd|dd} tj|d|d} || | ||d qWd |S1sMwY|S) z1Normalizes the Nancy meta data file to TTS formatnancyrrJr r "wavnrLrMN)r+r,r-r.r1findrfindr:) r(r;rPrQrCr'rRrButt_idr%rTrrrrs   rc Cstj||}g}t|dddE}|D]9}|drq|d}|d}|d} t|tr2| |vr2qtj|d|d d d } | || d | |d qWd|S1sXwY|S)z8Normalize the common voice meta data file to TTS format.rrJr r rKrtrclipsr z.mp3rLMCV_rMN) r+r,r-r. startswithr1rgrhrjr:) r(r;r<rQrCrRrBrSr%r'rTrrr common_voice-s*    rc CsDg}|st|ddd}n t|trtj||g}|D]l}tj|dd}t|dddP}|D]E}|d }|d} |dd ^} } } tj|| d | } tj| | d }|d }t|t rl| |vrlq4| ||d| |dq4Wdn1swYq|D]}tj |dsJd|dq|S)z+https://ai.google/tools/datasets/libri-tts/z/**/*trans.tsvTr^.rrrJr rKrY/rLrtLTTS_rMNr& [!] wav files don't exist - ) rrgstrr+r,r-basenamer1r.rhr:r9)r(rkr<rCr; _meta_filerRrBrS file_namer' chapter_idrY _root_pathrTr%rrrr libri_ttsCs>   $rc Kstj||}g}d}g}t|ddd<}|D]1}|d} tj|d| dd} tj| s8|| q| d } || | ||d qWdn1sSwYtd t |d |S) Nzturkish-femalerrJr r rWrrLr rMr}z# files skipped. They don't exist...) r+r,r-r.r1r4r9r:r3r0) r(r;rPrQrCr' skipped_filesrRrBrSrTr%rrrcustom_turkishgs"     rc Cstj||}g}t|ddd>}|D]2}|drq|d}tj||d}|d} |d} t|tr;| |vr;q|| || |d qWd |S1sQwY|S) zBRSpeech 3.0 betarrJr rr rrtrMN) r+r,r-r.rr1rgrhr:) r(r;r<rQrCrRrBrSrTr%rvrrrbrspeechzs&      rwav48_silence_trimmedmic1c Cs&d}g}ttj|dddd}|D]|}tj||tj\}} } | dd} t|tr5| |vr5qt |dd d } | d} Wd n1sMwY| d kretj||| | d |}ntj||| | d|d|}tj |r| | |d| |dqt d|q|S)uVCTK dataset v0.92. URL: https://datashare.ed.ac.uk/bitstream/handle/10283/3443/VCTK-Corpus-0.92.zip This dataset has 2 recordings per speaker that are annotated with ```mic1``` and ```mic2```. It is believed that (😄 ) ```mic1``` files are the same as the previous version of the dataset. mic1: Audio recorded using an omni-directional microphone (DPA 4035). Contains very low frequency noises. This is the same audio released in previous versions of VCTK: https://doi.org/10.7488/ds/1994 mic2: Audio recorded using a small diaphragm condenser microphone with very wide bandwidth (Sennheiser MKH 800). Two speakers, p280 and p315 had technical issues of the audio recordings using MKH 800. flactxt /**/*.txtTr^rrrrJr Np280z_mic1.rYVCTK_rMr)rr+r,r-relpathr1rrgrhr.r/r9r:r3)r(rk wavs_pathmicr<file_extrCr;rYrvrQfile_id file_textr%rTrrrvctks* " rwav48c Csg}ttj|dddd}|D]U}tj||tj\}}}|dd} t|tr3||vr3qt |ddd } | d} Wd n1sKwYtj|||| d } | | | d ||d q|S)z:homepages.inf.ed.ac.uk/jyamagis/release/VCTK-Corpus.tar.gzrrTr^rrrrJr NrL VCTK_old_rM) rr+r,r-rr1rrgrhr.r/r:) r(rkrr<rCr;rYrvrQrrr%rTrrrvctk_olds  rc Ksg}d}tj|d}t|ddd}|D]Z}tjdtj|vr)|dd}ntjtj|dtj|dd }tj|rptj|rpt |d d d }| d } Wdn1sawY| | |||dq|S)N synpaflexr`**/*.wavTr^wavrrLz.txtrrJr rrM) r+r,r-rrrjrdrr9r.r/r:) r( metafilesrPrCr' wav_filesrTrQrr%rrrrs"rtrainTc Csg}|}ttj||ddd}|D]b}tj||tj\}}} | dd} t|tr5||vr5qt |ddd} | d d } Wd n1sOwY|r_t t tj| r_qtj|||| d } || | d ||dq|S)z$ToDo: Refer the paper when availablerTr^rrrrJr  r`Nz.flacOB_rM)rr+r,r-rr1rrgrhr.readlinerjanymaprisdigitr:)r(rkignore_digits_sentencesr<rC split_dirr;rYrvrQrrr%rTrrr open_bibles" rc Csg}ttj||dddJ}|D]>}|d\}}|dd}|d^}} } tj|tj|d|| |d } t|trC||vrCq||| d ||d qWd|S1s[wY|S) zhttp://www.openslr.org/94/rrJr rKNrYaudiorLMLS_rM) r.r+r,r-r1rdrgrhr:) r(rkr<rCmetarBfiler%speakerbookrYrTrrrmlss$ "    rcKt||ddS)P :param meta_file Used only for consistency with load_tts_samples api 2 voxcel_idx _voxcel_xr(r;rPrrr voxceleb2rcKr)r1rrrrrr voxceleb1rrc Cs|dvsJ|dkr dnd}t|}|d|d}|jjdd|durHtt|d d d }d d |DWdS1sBwYnq|sd}g}|d} t| d|d|dD])} tt| jjj } | dsrJd} | | d| d|d| d|d7}q`tt|dd d }| d |Wdn1swY||krtd|d|tt|d d d }dd |DWdS1swYdS)N)rrri Bi@Bmetafile_voxcelebz.csvT)exist_okrrJr cSg|] }|dqSr r4r1rrrr &z_voxcel_x..rrzBuilding VoxCeleb z/ Meta file ... this needs to be done only once.)desctotalr|r z|voxcelrYrr wr`z7Found too few instances for Voxceleb. Should be around z, is: cSrrrrrrrr<r)rparentmkdirr.rr/r9rglobrstemrr:writer- ValueError) r(r;rexpected_count voxceleb_pathcache_tor>cnt meta_datarr,rvr%rrrrs> "   " $rc Cstj||}g}t|dddB}|D]6}|drq|d}tj||d}|d} |dd d } t|tr?| |vr?q| || | |d qWd |S1sUwY|S) zGeneric emotion datasetrrJr file_path,rr rtrr`)r&r'r"r(N) r+r,r-r.rr1rjrgrhr:) r(r;r<rQrCrRrBrSrTrv emotion_idrrremotion?s*     rr(r;returnc Kstj||}g}d}t|ddd*}|D]}|dd\}} tj|d|} || | ||dqWd |S1s?wY|S) aWNormalizes the Baker meta data file to TTS format Args: root_path (str): path to the baker dataset meta_file (str): name of the meta dataset containing names of wav to select and the transcript of the sentence Returns: List[List[str]]: List of (text, wav_path, speaker_name) associated with each sentences bakerrrJr rr clips_22rMN)r+r,r-r.rstripr1r:) r(r;rPrQrCr'rRrBwav_namer%wav_pathrrrrUs  rc Kstj||}g}d}t|ddd1}|D]%}|d}tj|d|dd} |d d d } || | ||d qWd |S1sFwY|S)zVJapanese single-speaker dataset from https://github.com/kaiidams/Kokoro-Speech-DatasetkokororrJr r rWrrLrt r`rMN)r+r,r-r.r1rjr:rOrrrris  rc Kr) zqKorean single-speaker dataset from https://www.kaggle.com/datasets/bryanpark/korean-single-speaker-speech-datasetkssrrJr r rrtrMNrNrOrrrrwrrc Kstj||}g}d}t|ddd*}|D]}|d}tj||d} |d} || | ||dqWd|S1s?wY|S) Nbel_ttsrrJr r rr rMrNrOrrrbel_tts_formatters  rr)NN)NrrN)NrN)rTN)-r+raxml.etree.ElementTreeetree ElementTreer~rpathlibrtypingrpandasr5rrGrHrIrVr\rqrsrwrxrzrrrrrrrrrrrrrrrrrrrrrrrrrsL      '& 3  $   /      #