|
> Training Environment: |
|
| > Backend: Torch |
|
| > Mixed precision: False |
|
| > Precision: float32 |
|
| > Current device: 0 |
|
| > Num. of GPUs: 1 |
|
| > Num. of CPUs: 128 |
|
| > Num. of Torch Threads: 1 |
|
| > Torch seed: 54321 |
|
| > Torch CUDNN: True |
|
| > Torch CUDNN deterministic: False |
|
| > Torch CUDNN benchmark: False |
|
| > Torch TF32 MatMul: True |
|
> Start Tensorboard: tensorboard --logdir=/checkpoints/GPT_XTTS_FT-December-23-2024_04+57PM-0000000 |
|
|
|
> Model has 532793243 parameters |
|
|
|
[4m[1m > EPOCH: 0/10[0m |
|
--> /checkpoints/GPT_XTTS_FT-December-23-2024_04+57PM-0000000 |
|
|
|
[1m > TRAINING (2024-12-23 16:57:09) [0m |
|
|
|
[1m --> TIME: 2024-12-23 16:57:12 -- STEP: 0/1935 -- GLOBAL_STEP: 0[0m |
|
| > loss_text_ce: 0.044483885169029236 (0.044483885169029236) |
|
| > loss_mel_ce: 2.8605117797851562 (2.8605117797851562) |
|
| > loss: 0.36312445998191833 (0.36312445998191833) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.7688 (0.7687897682189941) |
|
| > loader_time: 1.7344 (1.7343621253967285) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 16:57:38 -- STEP: 50/1935 -- GLOBAL_STEP: 50[0m |
|
| > loss_text_ce: 0.043564990162849426 (0.04478648766875267) |
|
| > loss_mel_ce: 2.838747978210449 (2.9473777341842657) |
|
| > loss: 0.3602891266345978 (0.37402052879333497) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2403 (0.3284364318847656) |
|
| > loader_time: 0.0387 (0.0431177806854248) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 16:58:04 -- STEP: 100/1935 -- GLOBAL_STEP: 100[0m |
|
| > loss_text_ce: 0.04516163095831871 (0.04485867124050855) |
|
| > loss_mel_ce: 3.0671980381011963 (2.959674265384674) |
|
| > loss: 0.38904497027397156 (0.37556661933660507) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2939 (0.32423117160797105) |
|
| > loader_time: 0.0447 (0.04224255561828614) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 16:58:30 -- STEP: 150/1935 -- GLOBAL_STEP: 150[0m |
|
| > loss_text_ce: 0.04512446001172066 (0.04483936612804731) |
|
| > loss_mel_ce: 2.9204883575439453 (2.960846746762594) |
|
| > loss: 0.3707016110420227 (0.37571076571941375) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3315 (0.32382096926371245) |
|
| > loader_time: 0.031 (0.04090922355651856) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 16:58:56 -- STEP: 200/1935 -- GLOBAL_STEP: 200[0m |
|
| > loss_text_ce: 0.045201342552900314 (0.04486061621457339) |
|
| > loss_mel_ce: 2.9472744464874268 (2.9612675857543955) |
|
| > loss: 0.3740594685077667 (0.37576602697372435) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3109 (0.32348613858222963) |
|
| > loader_time: 0.0392 (0.04018751621246336) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 16:59:22 -- STEP: 250/1935 -- GLOBAL_STEP: 250[0m |
|
| > loss_text_ce: 0.044838521629571915 (0.044834685340523726) |
|
| > loss_mel_ce: 3.000558376312256 (2.960930704116822) |
|
| > loss: 0.3806746006011963 (0.37572067487239835) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.271 (0.3232661046981812) |
|
| > loader_time: 0.0385 (0.039428794860839826) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 16:59:48 -- STEP: 300/1935 -- GLOBAL_STEP: 300[0m |
|
| > loss_text_ce: 0.045189566910266876 (0.04487013855328164) |
|
| > loss_mel_ce: 2.994903326034546 (2.9594988854726165) |
|
| > loss: 0.3800116181373596 (0.3755461287498474) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3411 (0.32350548267364515) |
|
| > loader_time: 0.0346 (0.038672723770141616) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:00:15 -- STEP: 350/1935 -- GLOBAL_STEP: 350[0m |
|
| > loss_text_ce: 0.044412922114133835 (0.044896405243447866) |
|
| > loss_mel_ce: 3.0054280757904053 (2.9593866041728436) |
|
| > loss: 0.38123011589050293 (0.3755353766679762) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3381 (0.32420502867017476) |
|
| > loader_time: 0.0332 (0.03820698874337332) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:00:41 -- STEP: 400/1935 -- GLOBAL_STEP: 400[0m |
|
| > loss_text_ce: 0.0448891706764698 (0.04490688295103611) |
|
| > loss_mel_ce: 2.914822578430176 (2.959149395227433) |
|
| > loss: 0.36996397376060486 (0.3755070353299378) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.295 (0.3251341903209688) |
|
| > loader_time: 0.0368 (0.0378065001964569) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:01:08 -- STEP: 450/1935 -- GLOBAL_STEP: 450[0m |
|
| > loss_text_ce: 0.04158417508006096 (0.04491182804107668) |
|
| > loss_mel_ce: 2.9100966453552246 (2.9579920000500155) |
|
| > loss: 0.36896011233329773 (0.37536297884252323) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3194 (0.3257565651999581) |
|
| > loader_time: 0.0351 (0.03748372660742862) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:01:33 -- STEP: 500/1935 -- GLOBAL_STEP: 500[0m |
|
| > loss_text_ce: 0.04489632323384285 (0.04489993647485973) |
|
| > loss_mel_ce: 2.99580454826355 (2.9583330082893378) |
|
| > loss: 0.38008761405944824 (0.3754041182398795) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2976 (0.32519007015228285) |
|
| > loader_time: 0.041 (0.03713997793197629) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:02:00 -- STEP: 550/1935 -- GLOBAL_STEP: 550[0m |
|
| > loss_text_ce: 0.04610965773463249 (0.04490645095028663) |
|
| > loss_mel_ce: 3.0559351444244385 (2.9582581550424756) |
|
| > loss: 0.3877556025981903 (0.375395575978539) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3418 (0.3253071186759256) |
|
| > loader_time: 0.0294 (0.03696944453499532) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:02:26 -- STEP: 600/1935 -- GLOBAL_STEP: 600[0m |
|
| > loss_text_ce: 0.04606836661696434 (0.0449127968090276) |
|
| > loss_mel_ce: 2.984401226043701 (2.959451797803244) |
|
| > loss: 0.3788087069988251 (0.3755455744763215) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2728 (0.32529951612154645) |
|
| > loader_time: 0.0323 (0.03679212609926858) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:02:52 -- STEP: 650/1935 -- GLOBAL_STEP: 650[0m |
|
| > loss_text_ce: 0.046455372124910355 (0.0449463545760283) |
|
| > loss_mel_ce: 2.956956624984741 (2.9593748767559354) |
|
| > loss: 0.3754265010356903 (0.375540154163654) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.273 (0.3254119003736056) |
|
| > loader_time: 0.0334 (0.03661203714517446) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:03:18 -- STEP: 700/1935 -- GLOBAL_STEP: 700[0m |
|
| > loss_text_ce: 0.04376790300011635 (0.04493723662836208) |
|
| > loss_mel_ce: 2.9872536659240723 (2.959900025640215) |
|
| > loss: 0.3788776993751526 (0.3756046579565319) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3457 (0.3256708598136902) |
|
| > loader_time: 0.0342 (0.036473490851266034) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:03:44 -- STEP: 750/1935 -- GLOBAL_STEP: 750[0m |
|
| > loss_text_ce: 0.04318978637456894 (0.04493158126374084) |
|
| > loss_mel_ce: 2.9480020999908447 (2.9588217693964642) |
|
| > loss: 0.373898983001709 (0.3754691691795983) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3475 (0.3254640951156617) |
|
| > loader_time: 0.0344 (0.03632860660552979) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:04:10 -- STEP: 800/1935 -- GLOBAL_STEP: 800[0m |
|
| > loss_text_ce: 0.04504064470529556 (0.04490937288850544) |
|
| > loss_mel_ce: 2.919872999191284 (2.9574720108509065) |
|
| > loss: 0.3706142008304596 (0.3752976733073591) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3177 (0.3252555200457573) |
|
| > loader_time: 0.0371 (0.03621471256017684) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:04:36 -- STEP: 850/1935 -- GLOBAL_STEP: 850[0m |
|
| > loss_text_ce: 0.04591979831457138 (0.04490521541851407) |
|
| > loss_mel_ce: 2.8287603855133057 (2.956520117310917) |
|
| > loss: 0.3593350350856781 (0.37517816690837624) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2846 (0.32510946975034827) |
|
| > loader_time: 0.0382 (0.036191204856423756) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:05:02 -- STEP: 900/1935 -- GLOBAL_STEP: 900[0m |
|
| > loss_text_ce: 0.04314867779612541 (0.04490984618663786) |
|
| > loss_mel_ce: 2.9945895671844482 (2.956415424876743) |
|
| > loss: 0.37971729040145874 (0.3751656592554515) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3439 (0.3252403163909912) |
|
| > loader_time: 0.0335 (0.03610556920369463) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:05:28 -- STEP: 950/1935 -- GLOBAL_STEP: 950[0m |
|
| > loss_text_ce: 0.04534834250807762 (0.044894759325604675) |
|
| > loss_mel_ce: 2.871727228164673 (2.9564287747834856) |
|
| > loss: 0.3646344542503357 (0.37516544219694625) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.31 (0.32503469768323395) |
|
| > loader_time: 0.0336 (0.03600689787613715) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:05:55 -- STEP: 1000/1935 -- GLOBAL_STEP: 1000[0m |
|
| > loss_text_ce: 0.045473579317331314 (0.04489901082217692) |
|
| > loss_mel_ce: 2.862858772277832 (2.9555772387981416) |
|
| > loss: 0.36354154348373413 (0.3750595315694808) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.325 (0.325159752368927) |
|
| > loader_time: 0.034 (0.035972031593322726) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:06:20 -- STEP: 1050/1935 -- GLOBAL_STEP: 1050[0m |
|
| > loss_text_ce: 0.042760007083415985 (0.044901976216407034) |
|
| > loss_mel_ce: 2.9882469177246094 (2.9553337181182133) |
|
| > loss: 0.37887585163116455 (0.37502946217854805) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3379 (0.32489866960616337) |
|
| > loader_time: 0.0343 (0.03591666698455808) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:06:46 -- STEP: 1100/1935 -- GLOBAL_STEP: 1100[0m |
|
| > loss_text_ce: 0.045095186680555344 (0.04488515588370235) |
|
| > loss_mel_ce: 2.965927839279175 (2.9548881194808265) |
|
| > loss: 0.3763778805732727 (0.37497165980664154) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3096 (0.32473051721399504) |
|
| > loader_time: 0.0337 (0.035885705947875934) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:07:12 -- STEP: 1150/1935 -- GLOBAL_STEP: 1150[0m |
|
| > loss_text_ce: 0.04436462000012398 (0.04488931904344455) |
|
| > loss_mel_ce: 2.9464094638824463 (2.954161406185315) |
|
| > loss: 0.37384676933288574 (0.374881341042726) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.335 (0.324757710747097) |
|
| > loader_time: 0.0358 (0.035801767473635414) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:07:38 -- STEP: 1200/1935 -- GLOBAL_STEP: 1200[0m |
|
| > loss_text_ce: 0.04622793197631836 (0.044881100471441934) |
|
| > loss_mel_ce: 2.925412654876709 (2.9545836546023665) |
|
| > loss: 0.3714550733566284 (0.37493309485415643) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2888 (0.3245633190870286) |
|
| > loader_time: 0.0431 (0.035729106465975366) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:08:04 -- STEP: 1250/1935 -- GLOBAL_STEP: 1250[0m |
|
| > loss_text_ce: 0.045541565865278244 (0.0448739756256342) |
|
| > loss_mel_ce: 2.95540452003479 (2.9541411237716666) |
|
| > loss: 0.3751182556152344 (0.3748768878698352) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3427 (0.32458571605682396) |
|
| > loader_time: 0.0341 (0.03569356136322016) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:08:30 -- STEP: 1300/1935 -- GLOBAL_STEP: 1300[0m |
|
| > loss_text_ce: 0.04549604281783104 (0.04487008124016801) |
|
| > loss_mel_ce: 2.9381306171417236 (2.95442983150482) |
|
| > loss: 0.372953325510025 (0.3749124895150848) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2759 (0.3246268481474659) |
|
| > loader_time: 0.0334 (0.035667665371527976) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:08:57 -- STEP: 1350/1935 -- GLOBAL_STEP: 1350[0m |
|
| > loss_text_ce: 0.04626189172267914 (0.0448593590684511) |
|
| > loss_mel_ce: 2.9610695838928223 (2.9545009697808142) |
|
| > loss: 0.37591642141342163 (0.37492004151697567) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2988 (0.32480813838817457) |
|
| > loader_time: 0.026 (0.03560820667831982) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:09:23 -- STEP: 1400/1935 -- GLOBAL_STEP: 1400[0m |
|
| > loss_text_ce: 0.0427958108484745 (0.044856468135757104) |
|
| > loss_mel_ce: 2.913496732711792 (2.954512947286876) |
|
| > loss: 0.3695365786552429 (0.3749211772850588) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2875 (0.32507021171706085) |
|
| > loader_time: 0.0345 (0.03557604960032869) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:09:50 -- STEP: 1450/1935 -- GLOBAL_STEP: 1450[0m |
|
| > loss_text_ce: 0.04463299736380577 (0.044854371432086544) |
|
| > loss_mel_ce: 3.012481689453125 (2.954767877480077) |
|
| > loss: 0.38213932514190674 (0.3749527815087095) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2945 (0.3251334884248933) |
|
| > loader_time: 0.0331 (0.035568811153543385) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:10:15 -- STEP: 1500/1935 -- GLOBAL_STEP: 1500[0m |
|
| > loss_text_ce: 0.04319855943322182 (0.04484587074071172) |
|
| > loss_mel_ce: 2.9368014335632324 (2.954750808715818) |
|
| > loss: 0.3725000023841858 (0.374949585318566) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2973 (0.3249499877293907) |
|
| > loader_time: 0.0336 (0.035499076843261715) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:10:41 -- STEP: 1550/1935 -- GLOBAL_STEP: 1550[0m |
|
| > loss_text_ce: 0.04549115523695946 (0.04483618650705586) |
|
| > loss_mel_ce: 3.024620294570923 (2.954513874207771) |
|
| > loss: 0.3837639391422272 (0.3749187579654884) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3756 (0.3249557990412559) |
|
| > loader_time: 0.0374 (0.03544947608824695) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:11:08 -- STEP: 1600/1935 -- GLOBAL_STEP: 1600[0m |
|
| > loss_text_ce: 0.04424729570746422 (0.04482093332801017) |
|
| > loss_mel_ce: 3.033457040786743 (2.9545115607976893) |
|
| > loss: 0.3847130537033081 (0.37491656213998853) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3262 (0.32496736794710146) |
|
| > loader_time: 0.0347 (0.035435314029455135) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:11:34 -- STEP: 1650/1935 -- GLOBAL_STEP: 1650[0m |
|
| > loss_text_ce: 0.04383805766701698 (0.04480838916066924) |
|
| > loss_mel_ce: 2.9730749130249023 (2.9543734972404687) |
|
| > loss: 0.37711411714553833 (0.37489773609421523) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3098 (0.3250504266854487) |
|
| > loader_time: 0.0338 (0.035418744520707514) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:12:00 -- STEP: 1700/1935 -- GLOBAL_STEP: 1700[0m |
|
| > loss_text_ce: 0.04476361349225044 (0.044804771790171394) |
|
| > loss_mel_ce: 2.910047769546509 (2.9540041588334454) |
|
| > loss: 0.36935141682624817 (0.3748511165555791) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3527 (0.3249554489640627) |
|
| > loader_time: 0.0356 (0.03541813864427449) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:12:25 -- STEP: 1750/1935 -- GLOBAL_STEP: 1750[0m |
|
| > loss_text_ce: 0.043716829270124435 (0.04480193536196438) |
|
| > loss_mel_ce: 3.089958667755127 (2.953727010318209) |
|
| > loss: 0.39170944690704346 (0.37481611847877555) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.345 (0.3246730241775511) |
|
| > loader_time: 0.0353 (0.03539234229496542) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:12:52 -- STEP: 1800/1935 -- GLOBAL_STEP: 1800[0m |
|
| > loss_text_ce: 0.04568669572472572 (0.044790884968307305) |
|
| > loss_mel_ce: 2.9359045028686523 (2.9536996703677687) |
|
| > loss: 0.37269890308380127 (0.3748113196425973) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.295 (0.32469122648239124) |
|
| > loader_time: 0.0332 (0.03537253393067248) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:13:18 -- STEP: 1850/1935 -- GLOBAL_STEP: 1850[0m |
|
| > loss_text_ce: 0.04428812488913536 (0.044783445683282797) |
|
| > loss_mel_ce: 3.0728726387023926 (2.953527303644127) |
|
| > loss: 0.3896450996398926 (0.3747888439249353) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3534 (0.32466826941515936) |
|
| > loader_time: 0.035 (0.03536816957834599) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:13:44 -- STEP: 1900/1935 -- GLOBAL_STEP: 1900[0m |
|
| > loss_text_ce: 0.04465319961309433 (0.044780343757256105) |
|
| > loss_mel_ce: 2.908193588256836 (2.9530365997866563) |
|
| > loss: 0.36910584568977356 (0.3747271182192004) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3436 (0.32471458083704885) |
|
| > loader_time: 0.035 (0.03535993990145225) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time: 0.11557478013664783 [0m(+0) |
|
| > avg_loss_text_ce: 0.04289325753772499 [0m(+0) |
|
| > avg_loss_mel_ce: 3.0005661309367486 [0m(+0) |
|
| > avg_loss: 3.0434593913531076 [0m(+0) |
|
|
|
|
|
[4m[1m > EPOCH: 1/10[0m |
|
--> /checkpoints/GPT_XTTS_FT-December-23-2024_04+57PM-0000000 |
|
|
|
[1m > TRAINING (2024-12-23 17:15:10) [0m |
|
|
|
[1m --> TIME: 2024-12-23 17:15:18 -- STEP: 15/1935 -- GLOBAL_STEP: 1950[0m |
|
| > loss_text_ce: 0.04321984201669693 (0.04406609411040942) |
|
| > loss_mel_ce: 2.92112135887146 (2.9722349325815838) |
|
| > loss: 0.37054264545440674 (0.37703762849171957) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3665 (0.3287209033966064) |
|
| > loader_time: 0.0248 (0.02239076296488444) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:15:43 -- STEP: 65/1935 -- GLOBAL_STEP: 2000[0m |
|
| > loss_text_ce: 0.046198878437280655 (0.04447965386968394) |
|
| > loss_mel_ce: 2.9545018672943115 (2.9491459809816796) |
|
| > loss: 0.37508758902549744 (0.3742032060256371) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3126 (0.3212986946105957) |
|
| > loader_time: 0.0206 (0.021251392364501954) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:16:08 -- STEP: 115/1935 -- GLOBAL_STEP: 2050[0m |
|
| > loss_text_ce: 0.04422479867935181 (0.044426388682230665) |
|
| > loss_mel_ce: 2.9806714057922363 (2.947338910724805) |
|
| > loss: 0.3781120181083679 (0.3739706638066665) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2948 (0.31940855357957915) |
|
| > loader_time: 0.0224 (0.020652737824813187) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:16:34 -- STEP: 165/1935 -- GLOBAL_STEP: 2100[0m |
|
| > loss_text_ce: 0.04502017796039581 (0.04437728108781756) |
|
| > loss_mel_ce: 2.9583587646484375 (2.947598130775219) |
|
| > loss: 0.3754223585128784 (0.37399692770206566) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3453 (0.3223356926079952) |
|
| > loader_time: 0.0345 (0.020650433049057483) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:17:00 -- STEP: 215/1935 -- GLOBAL_STEP: 2150[0m |
|
| > loss_text_ce: 0.043103624135255814 (0.044394641265619636) |
|
| > loss_mel_ce: 2.926520586013794 (2.946181483601413) |
|
| > loss: 0.3712030351161957 (0.3738220164942187) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.361 (0.32317773020544716) |
|
| > loader_time: 0.0345 (0.023563119977019566) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:17:26 -- STEP: 265/1935 -- GLOBAL_STEP: 2200[0m |
|
| > loss_text_ce: 0.04323988035321236 (0.04441979583704247) |
|
| > loss_mel_ce: 2.886981248855591 (2.9456241796601477) |
|
| > loss: 0.36627763509750366 (0.3737554973026492) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3437 (0.32366010917807525) |
|
| > loader_time: 0.0208 (0.02417078917881229) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:17:51 -- STEP: 315/1935 -- GLOBAL_STEP: 2250[0m |
|
| > loss_text_ce: 0.04259290546178818 (0.04442305684326187) |
|
| > loss_mel_ce: 2.8937344551086426 (2.9452054856315466) |
|
| > loss: 0.3670409321784973 (0.3737035681330968) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3094 (0.32284304225255583) |
|
| > loader_time: 0.0328 (0.02380982959081257) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:18:17 -- STEP: 365/1935 -- GLOBAL_STEP: 2300[0m |
|
| > loss_text_ce: 0.04438846930861473 (0.04445146599861045) |
|
| > loss_mel_ce: 2.904825210571289 (2.9451090192141574) |
|
| > loss: 0.36865171790122986 (0.37369506097819716) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3385 (0.3232579296582367) |
|
| > loader_time: 0.032 (0.025107052554822956) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:18:43 -- STEP: 415/1935 -- GLOBAL_STEP: 2350[0m |
|
| > loss_text_ce: 0.04479111731052399 (0.0444454632879022) |
|
| > loss_mel_ce: 2.949154853820801 (2.94413002255451) |
|
| > loss: 0.37424325942993164 (0.3735719358346549) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3631 (0.3233003300356578) |
|
| > loader_time: 0.036 (0.02610579973243806) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:19:09 -- STEP: 465/1935 -- GLOBAL_STEP: 2400[0m |
|
| > loss_text_ce: 0.04662776738405228 (0.044463033894056946) |
|
| > loss_mel_ce: 2.942779779434204 (2.943855966034754) |
|
| > loss: 0.3736759424209595 (0.37353987520740883) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3161 (0.3236927750290081) |
|
| > loader_time: 0.033 (0.02702802022298178) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:19:35 -- STEP: 515/1935 -- GLOBAL_STEP: 2450[0m |
|
| > loss_text_ce: 0.04236850142478943 (0.04443747209547796) |
|
| > loss_mel_ce: 3.030348777770996 (2.944055368830855) |
|
| > loss: 0.3840896487236023 (0.37356160520349896) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3441 (0.32405505458128103) |
|
| > loader_time: 0.0341 (0.027766938348418306) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:20:01 -- STEP: 565/1935 -- GLOBAL_STEP: 2500[0m |
|
| > loss_text_ce: 0.04435889050364494 (0.04444205445789663) |
|
| > loss_mel_ce: 2.9815542697906494 (2.9449402479998814) |
|
| > loss: 0.37823915481567383 (0.3736727879638167) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3197 (0.32401671240815016) |
|
| > loader_time: 0.0348 (0.028280991157599272) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:20:27 -- STEP: 615/1935 -- GLOBAL_STEP: 2550[0m |
|
| > loss_text_ce: 0.04635847359895706 (0.044445387195280926) |
|
| > loss_mel_ce: 2.9915034770965576 (2.94393734040299) |
|
| > loss: 0.37973275780677795 (0.3735478410875894) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3663 (0.32360926217179947) |
|
| > loader_time: 0.0345 (0.02874353649170419) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:20:53 -- STEP: 665/1935 -- GLOBAL_STEP: 2600[0m |
|
| > loss_text_ce: 0.0452701635658741 (0.044455336146336795) |
|
| > loss_mel_ce: 2.9827492237091064 (2.943074563750646) |
|
| > loss: 0.37850242853164673 (0.37344123756975145) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3325 (0.3234328255617529) |
|
| > loader_time: 0.0341 (0.029113205931240466) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:21:19 -- STEP: 715/1935 -- GLOBAL_STEP: 2650[0m |
|
| > loss_text_ce: 0.045308131724596024 (0.04444773067961212) |
|
| > loss_mel_ce: 2.9823012351989746 (2.942707491921377) |
|
| > loss: 0.3784511685371399 (0.37339440296579884) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3378 (0.3234812806536267) |
|
| > loader_time: 0.034 (0.029454298953076354) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:21:45 -- STEP: 765/1935 -- GLOBAL_STEP: 2700[0m |
|
| > loss_text_ce: 0.045404884964227676 (0.044460548431265644) |
|
| > loss_mel_ce: 2.9015398025512695 (2.942762091580558) |
|
| > loss: 0.36836808919906616 (0.37340283019869913) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3001 (0.32331070619470953) |
|
| > loader_time: 0.0335 (0.029704074921950804) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:22:11 -- STEP: 815/1935 -- GLOBAL_STEP: 2750[0m |
|
| > loss_text_ce: 0.04480995982885361 (0.044448223774052886) |
|
| > loss_mel_ce: 2.820081949234009 (2.942485987189356) |
|
| > loss: 0.35811150074005127 (0.37336677663896695) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2535 (0.32338584595662684) |
|
| > loader_time: 0.036 (0.03002303217086325) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:22:37 -- STEP: 865/1935 -- GLOBAL_STEP: 2800[0m |
|
| > loss_text_ce: 0.043683651834726334 (0.04444181370425091) |
|
| > loss_mel_ce: 2.9223992824554443 (2.942134072188007) |
|
| > loss: 0.37076038122177124 (0.37332198602615735) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2967 (0.3232897617913395) |
|
| > loader_time: 0.0322 (0.030274559307649655) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:23:03 -- STEP: 915/1935 -- GLOBAL_STEP: 2850[0m |
|
| > loss_text_ce: 0.044956374913454056 (0.04443399022674305) |
|
| > loss_mel_ce: 2.911391496658325 (2.9420512191584844) |
|
| > loss: 0.36954349279403687 (0.3733106513818103) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3542 (0.32362499315230575) |
|
| > loader_time: 0.0336 (0.030498132028214928) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:23:29 -- STEP: 965/1935 -- GLOBAL_STEP: 2900[0m |
|
| > loss_text_ce: 0.043023962527513504 (0.044426210097235144) |
|
| > loss_mel_ce: 2.9166533946990967 (2.9415897804220714) |
|
| > loss: 0.36995968222618103 (0.3732519991657275) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3186 (0.32361154556274413) |
|
| > loader_time: 0.0262 (0.030695564635677047) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:23:55 -- STEP: 1015/1935 -- GLOBAL_STEP: 2950[0m |
|
| > loss_text_ce: 0.043939027935266495 (0.04441442696258357) |
|
| > loss_mel_ce: 2.956439256668091 (2.942131528243642) |
|
| > loss: 0.3750472962856293 (0.3733182448178088) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3774 (0.32366501568573447) |
|
| > loader_time: 0.0356 (0.030845686719922605) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:24:21 -- STEP: 1065/1935 -- GLOBAL_STEP: 3000[0m |
|
| > loss_text_ce: 0.04444253817200661 (0.04441914151951744) |
|
| > loss_mel_ce: 3.0356061458587646 (2.9422186215718584) |
|
| > loss: 0.3850060999393463 (0.3733297208105452) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3117 (0.3236100105052822) |
|
| > loader_time: 0.0334 (0.03098577065087261) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:24:47 -- STEP: 1115/1935 -- GLOBAL_STEP: 3050[0m |
|
| > loss_text_ce: 0.04213308170437813 (0.044411102618871816) |
|
| > loss_mel_ce: 2.95792555809021 (2.942157965604499) |
|
| > loss: 0.3750073313713074 (0.3733211339589189) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2983 (0.3234817744370532) |
|
| > loader_time: 0.0352 (0.0311584239583379) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:25:13 -- STEP: 1165/1935 -- GLOBAL_STEP: 3100[0m |
|
| > loss_text_ce: 0.04162346571683884 (0.04438688440384275) |
|
| > loss_mel_ce: 2.875277280807495 (2.9420062259543087) |
|
| > loss: 0.3646125793457031 (0.3732991392008734) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3358 (0.32359927263382676) |
|
| > loader_time: 0.0343 (0.03131783796482332) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:25:39 -- STEP: 1215/1935 -- GLOBAL_STEP: 3150[0m |
|
| > loss_text_ce: 0.04294915124773979 (0.0443816285841014) |
|
| > loss_mel_ce: 3.026026964187622 (2.942072047716305) |
|
| > loss: 0.38362202048301697 (0.37330670999401344) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3807 (0.3238386530935028) |
|
| > loader_time: 0.0352 (0.03143067301055531) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:26:06 -- STEP: 1265/1935 -- GLOBAL_STEP: 3200[0m |
|
| > loss_text_ce: 0.0469500869512558 (0.044385968080975814) |
|
| > loss_mel_ce: 2.9654407501220703 (2.9419912852788626) |
|
| > loss: 0.3765488564968109 (0.37329715711797135) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3097 (0.3238825694374414) |
|
| > loader_time: 0.0351 (0.03159036655199861) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:26:32 -- STEP: 1315/1935 -- GLOBAL_STEP: 3250[0m |
|
| > loss_text_ce: 0.043806660920381546 (0.04438744161858761) |
|
| > loss_mel_ce: 2.945580005645752 (2.9420731800137365) |
|
| > loss: 0.37367331981658936 (0.37330757812402043) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3387 (0.32411517698048614) |
|
| > loader_time: 0.035 (0.03136811709675956) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:26:57 -- STEP: 1365/1935 -- GLOBAL_STEP: 3300[0m |
|
| > loss_text_ce: 0.04604404792189598 (0.0443905130519972) |
|
| > loss_mel_ce: 2.974860906600952 (2.9413578676216763) |
|
| > loss: 0.3776131272315979 (0.3732185480140501) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3322 (0.32404633459154036) |
|
| > loader_time: 0.0342 (0.031496517474834784) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:27:24 -- STEP: 1415/1935 -- GLOBAL_STEP: 3350[0m |
|
| > loss_text_ce: 0.04665907844901085 (0.04438555810350414) |
|
| > loss_mel_ce: 2.8232851028442383 (2.9412348403526294) |
|
| > loss: 0.35874301195144653 (0.37320255017954523) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2974 (0.32416980291845066) |
|
| > loader_time: 0.0335 (0.031612599443631144) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:27:50 -- STEP: 1465/1935 -- GLOBAL_STEP: 3400[0m |
|
| > loss_text_ce: 0.044502142816782 (0.04437994955698785) |
|
| > loss_mel_ce: 2.950518846511841 (2.9417055722389622) |
|
| > loss: 0.37437763810157776 (0.3732606905719116) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3117 (0.32436028503313774) |
|
| > loader_time: 0.0351 (0.03173911221605112) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:28:16 -- STEP: 1515/1935 -- GLOBAL_STEP: 3450[0m |
|
| > loss_text_ce: 0.04263552278280258 (0.04437502935627113) |
|
| > loss_mel_ce: 2.9208686351776123 (2.941123894417641) |
|
| > loss: 0.37043800950050354 (0.3731873658033878) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3216 (0.3243246884235846) |
|
| > loader_time: 0.0338 (0.03184042763788705) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:28:42 -- STEP: 1565/1935 -- GLOBAL_STEP: 3500[0m |
|
| > loss_text_ce: 0.042945001274347305 (0.04437039825624932) |
|
| > loss_mel_ce: 2.98081111907959 (2.941180943376341) |
|
| > loss: 0.37796950340270996 (0.3731939180209611) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3162 (0.3242434807859667) |
|
| > loader_time: 0.0334 (0.031909235597799346) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:29:08 -- STEP: 1615/1935 -- GLOBAL_STEP: 3550[0m |
|
| > loss_text_ce: 0.043702758848667145 (0.04436936563474844) |
|
| > loss_mel_ce: 2.954244613647461 (2.9409626284619983) |
|
| > loss: 0.37474343180656433 (0.37316649962135856) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3664 (0.32412937196784686) |
|
| > loader_time: 0.0358 (0.031990559698996474) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:29:34 -- STEP: 1665/1935 -- GLOBAL_STEP: 3600[0m |
|
| > loss_text_ce: 0.04123224690556526 (0.044354184857896856) |
|
| > loss_mel_ce: 2.8590970039367676 (2.941052062089022) |
|
| > loss: 0.36254116892814636 (0.37317578121348516) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3434 (0.3241089832317361) |
|
| > loader_time: 0.0345 (0.03206353660102355) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:30:00 -- STEP: 1715/1935 -- GLOBAL_STEP: 3650[0m |
|
| > loss_text_ce: 0.043936677277088165 (0.04435010713364918) |
|
| > loss_mel_ce: 3.0190670490264893 (2.9410016782777313) |
|
| > loss: 0.3828754723072052 (0.37316897351609696) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3364 (0.3241806113685183) |
|
| > loader_time: 0.0331 (0.0321272374589658) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:30:26 -- STEP: 1765/1935 -- GLOBAL_STEP: 3700[0m |
|
| > loss_text_ce: 0.04599007964134216 (0.04434745880978303) |
|
| > loss_mel_ce: 2.888947010040283 (2.9411499308459153) |
|
| > loss: 0.3668671250343323 (0.37318717401696305) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3421 (0.3242564806519417) |
|
| > loader_time: 0.0358 (0.032206814417420244) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:30:52 -- STEP: 1815/1935 -- GLOBAL_STEP: 3750[0m |
|
| > loss_text_ce: 0.0446612685918808 (0.044346686764607415) |
|
| > loss_mel_ce: 2.9182896614074707 (2.9412310577949556) |
|
| > loss: 0.3703688681125641 (0.3731972184391387) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4047 (0.3240023186055751) |
|
| > loader_time: 0.045 (0.032279438749160935) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:31:18 -- STEP: 1865/1935 -- GLOBAL_STEP: 3800[0m |
|
| > loss_text_ce: 0.045070141553878784 (0.04434903543292677) |
|
| > loss_mel_ce: 2.9578938484191895 (2.941491794330506) |
|
| > loss: 0.37537050247192383 (0.37323010410444324) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3362 (0.32416715110594696) |
|
| > loader_time: 0.0358 (0.032336992616627834) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:31:44 -- STEP: 1915/1935 -- GLOBAL_STEP: 3850[0m |
|
| > loss_text_ce: 0.04336615279316902 (0.044349968723233005) |
|
| > loss_mel_ce: 2.989316463470459 (2.9415236108296856) |
|
| > loss: 0.37908533215522766 (0.3732341978475255) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3077 (0.32420633019728673) |
|
| > loader_time: 0.0341 (0.03240998307967618) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[91m 0.11609860381694753 [0m(+0.0005238236802996982) |
|
| > avg_loss_text_ce:[92m 0.04246167402074794 [0m(-0.0004315835169770438) |
|
| > avg_loss_mel_ce:[92m 2.995791206456194 [0m(-0.00477492448055461) |
|
| > avg_loss:[92m 3.038252875058339 [0m(-0.005206516294768537) |
|
|
|
|
|
[4m[1m > EPOCH: 2/10[0m |
|
--> /checkpoints/GPT_XTTS_FT-December-23-2024_04+57PM-0000000 |
|
|
|
[1m > TRAINING (2024-12-23 17:32:43) [0m |
|
|
|
[1m --> TIME: 2024-12-23 17:32:59 -- STEP: 30/1935 -- GLOBAL_STEP: 3900[0m |
|
| > loss_text_ce: 0.04341791570186615 (0.044189397369821866) |
|
| > loss_mel_ce: 2.9654664993286133 (2.945516554514567) |
|
| > loss: 0.3761105537414551 (0.37371324400107064) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.314 (0.3200124502182007) |
|
| > loader_time: 0.034 (0.03344808419545491) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:33:25 -- STEP: 80/1935 -- GLOBAL_STEP: 3950[0m |
|
| > loss_text_ce: 0.04497287794947624 (0.04433515546843409) |
|
| > loss_mel_ce: 2.957184076309204 (2.9430600404739384) |
|
| > loss: 0.3752696216106415 (0.3734243996441364) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3234 (0.32481261193752287) |
|
| > loader_time: 0.0322 (0.03372655808925628) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:33:52 -- STEP: 130/1935 -- GLOBAL_STEP: 4000[0m |
|
| > loss_text_ce: 0.044895708560943604 (0.04425700857089115) |
|
| > loss_mel_ce: 2.984452724456787 (2.9413063085996187) |
|
| > loss: 0.37866854667663574 (0.3731954143597529) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.314 (0.3259014313037579) |
|
| > loader_time: 0.0332 (0.03372816856090839) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:34:18 -- STEP: 180/1935 -- GLOBAL_STEP: 4050[0m |
|
| > loss_text_ce: 0.0420864038169384 (0.04412814246283637) |
|
| > loss_mel_ce: 2.9545187950134277 (2.9402738293011983) |
|
| > loss: 0.3745756447315216 (0.373050245973799) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3147 (0.32662256028917097) |
|
| > loader_time: 0.0332 (0.033698532316419776) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:34:44 -- STEP: 230/1935 -- GLOBAL_STEP: 4100[0m |
|
| > loss_text_ce: 0.044340576976537704 (0.04418301794516004) |
|
| > loss_mel_ce: 2.9219346046447754 (2.9396995368211165) |
|
| > loss: 0.3707844018936157 (0.3729853192101354) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3107 (0.32597690872524093) |
|
| > loader_time: 0.0333 (0.033588615707729134) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:35:10 -- STEP: 280/1935 -- GLOBAL_STEP: 4150[0m |
|
| > loss_text_ce: 0.04329419881105423 (0.04415362534512367) |
|
| > loss_mel_ce: 2.898226737976074 (2.9396020667893548) |
|
| > loss: 0.3676901161670685 (0.3729694616581712) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2856 (0.325630202463695) |
|
| > loader_time: 0.0333 (0.033263913222721604) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:35:36 -- STEP: 330/1935 -- GLOBAL_STEP: 4200[0m |
|
| > loss_text_ce: 0.042474567890167236 (0.04418748207390309) |
|
| > loss_mel_ce: 2.8911101818084717 (2.939602882211859) |
|
| > loss: 0.36669808626174927 (0.3729737955512423) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3103 (0.3251933986490424) |
|
| > loader_time: 0.023 (0.03310440453616052) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:36:02 -- STEP: 380/1935 -- GLOBAL_STEP: 4250[0m |
|
| > loss_text_ce: 0.045119114220142365 (0.0441745481796955) |
|
| > loss_mel_ce: 2.8551342487335205 (2.936534281780845) |
|
| > loss: 0.3625316619873047 (0.3725886033553827) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3528 (0.32466852664947526) |
|
| > loader_time: 0.0339 (0.033089152135347036) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:36:28 -- STEP: 430/1935 -- GLOBAL_STEP: 4300[0m |
|
| > loss_text_ce: 0.043731722980737686 (0.04417124969321629) |
|
| > loss_mel_ce: 2.962904930114746 (2.9362936019897456) |
|
| > loss: 0.3758295774459839 (0.37255810623945196) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3444 (0.3251894047093948) |
|
| > loader_time: 0.0344 (0.03309745622235672) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:36:54 -- STEP: 480/1935 -- GLOBAL_STEP: 4350[0m |
|
| > loss_text_ce: 0.04328557476401329 (0.044157062494196014) |
|
| > loss_mel_ce: 2.8921194076538086 (2.9364977578322087) |
|
| > loss: 0.3669256269931793 (0.372581852413714) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2957 (0.32533434033393877) |
|
| > loader_time: 0.033 (0.03307805508375168) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:37:20 -- STEP: 530/1935 -- GLOBAL_STEP: 4400[0m |
|
| > loss_text_ce: 0.04322011396288872 (0.04414445936398687) |
|
| > loss_mel_ce: 2.9705259799957275 (2.937121638711894) |
|
| > loss: 0.3767182528972626 (0.3726582621066076) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3465 (0.3249259719308818) |
|
| > loader_time: 0.0338 (0.03312606991461988) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:37:46 -- STEP: 580/1935 -- GLOBAL_STEP: 4450[0m |
|
| > loss_text_ce: 0.04399731010198593 (0.04411472252345292) |
|
| > loss_mel_ce: 3.069550037384033 (2.936807247276963) |
|
| > loss: 0.38919341564178467 (0.37261524601229323) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3137 (0.3251825057227038) |
|
| > loader_time: 0.0334 (0.033138492600671185) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:38:12 -- STEP: 630/1935 -- GLOBAL_STEP: 4500[0m |
|
| > loss_text_ce: 0.04395921155810356 (0.04411831015040951) |
|
| > loss_mel_ce: 2.9481382369995117 (2.9374215288767727) |
|
| > loss: 0.37401217222213745 (0.3726924796899161) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.271 (0.3251663189085705) |
|
| > loader_time: 0.0323 (0.03317640887366401) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:38:38 -- STEP: 680/1935 -- GLOBAL_STEP: 4550[0m |
|
| > loss_text_ce: 0.043815210461616516 (0.044102028992903615) |
|
| > loss_mel_ce: 2.9228124618530273 (2.9370282530784597) |
|
| > loss: 0.37082844972610474 (0.37264128511442873) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3243 (0.3251653439858382) |
|
| > loader_time: 0.035 (0.03322549883057092) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:39:04 -- STEP: 730/1935 -- GLOBAL_STEP: 4600[0m |
|
| > loss_text_ce: 0.04278935492038727 (0.044094306817405886) |
|
| > loss_mel_ce: 2.8240864276885986 (2.9363930372342657) |
|
| > loss: 0.3583594858646393 (0.3725609176779448) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2868 (0.3245516261009322) |
|
| > loader_time: 0.0337 (0.033222895125820225) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:39:30 -- STEP: 780/1935 -- GLOBAL_STEP: 4650[0m |
|
| > loss_text_ce: 0.04384191334247589 (0.044080468926292235) |
|
| > loss_mel_ce: 3.0165841579437256 (2.93636033626703) |
|
| > loss: 0.38255324959754944 (0.37255510030648664) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3129 (0.32458580273848314) |
|
| > loader_time: 0.0334 (0.033267422211475876) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:39:56 -- STEP: 830/1935 -- GLOBAL_STEP: 4700[0m |
|
| > loss_text_ce: 0.04291428625583649 (0.044069168976990605) |
|
| > loss_mel_ce: 2.967322826385498 (2.935659321532191) |
|
| > loss: 0.37627965211868286 (0.37246606109372116) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3149 (0.3248581291681311) |
|
| > loader_time: 0.0339 (0.03333848643015665) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:40:23 -- STEP: 880/1935 -- GLOBAL_STEP: 4750[0m |
|
| > loss_text_ce: 0.04247937723994255 (0.04407252606258477) |
|
| > loss_mel_ce: 2.9001214504241943 (2.9359927667812857) |
|
| > loss: 0.36782509088516235 (0.3725081613456662) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3134 (0.3249583333730696) |
|
| > loader_time: 0.0353 (0.03340373093431644) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:40:48 -- STEP: 930/1935 -- GLOBAL_STEP: 4800[0m |
|
| > loss_text_ce: 0.044296253472566605 (0.04407975328785761) |
|
| > loss_mel_ce: 2.985511302947998 (2.935280079226339) |
|
| > loss: 0.37872594594955444 (0.3724199788865224) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3203 (0.3246514412664597) |
|
| > loader_time: 0.0347 (0.03347121233581216) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:41:14 -- STEP: 980/1935 -- GLOBAL_STEP: 4850[0m |
|
| > loss_text_ce: 0.04277478903532028 (0.04406973097701465) |
|
| > loss_mel_ce: 2.9244823455810547 (2.9357281607024515) |
|
| > loss: 0.37090712785720825 (0.372474736340192) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3205 (0.3243691597666058) |
|
| > loader_time: 0.0347 (0.033543315712286546) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:41:40 -- STEP: 1030/1935 -- GLOBAL_STEP: 4900[0m |
|
| > loss_text_ce: 0.04499991238117218 (0.04406548750415013) |
|
| > loss_mel_ce: 2.8674123287200928 (2.9350035639642504) |
|
| > loss: 0.3640515208244324 (0.372383631317361) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3182 (0.32418359136118463) |
|
| > loader_time: 0.0357 (0.0335991195104655) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:42:06 -- STEP: 1080/1935 -- GLOBAL_STEP: 4950[0m |
|
| > loss_text_ce: 0.04304782673716545 (0.0440594437842568) |
|
| > loss_mel_ce: 2.9947474002838135 (2.935088677538766) |
|
| > loss: 0.3797244131565094 (0.3723935150161937) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.355 (0.3243063591144701) |
|
| > loader_time: 0.0363 (0.033673629495832684) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:42:33 -- STEP: 1130/1935 -- GLOBAL_STEP: 5000[0m |
|
| > loss_text_ce: 0.044562943279743195 (0.04404443989351264) |
|
| > loss_mel_ce: 2.8220973014831543 (2.9352584113061964) |
|
| > loss: 0.3583325445652008 (0.3724128563319687) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2932 (0.3243656382096551) |
|
| > loader_time: 0.0335 (0.033715570922446474) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:42:59 -- STEP: 1180/1935 -- GLOBAL_STEP: 5050[0m |
|
| > loss_text_ce: 0.04230651631951332 (0.04403960855077895) |
|
| > loss_mel_ce: 3.0174007415771484 (2.935197370537257) |
|
| > loss: 0.38246339559555054 (0.3724046223496986) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2704 (0.32427862619949593) |
|
| > loader_time: 0.0336 (0.033749805870702726) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:43:25 -- STEP: 1230/1935 -- GLOBAL_STEP: 5100[0m |
|
| > loss_text_ce: 0.044536758214235306 (0.04403711497723094) |
|
| > loss_mel_ce: 2.920111656188965 (2.934722778855299) |
|
| > loss: 0.3705810606479645 (0.3723449865976968) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3453 (0.32423989753412963) |
|
| > loader_time: 0.0353 (0.03380954653266967) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:43:51 -- STEP: 1280/1935 -- GLOBAL_STEP: 5150[0m |
|
| > loss_text_ce: 0.044393327087163925 (0.044042741725570586) |
|
| > loss_mel_ce: 2.930687427520752 (2.9350166834890845) |
|
| > loss: 0.3718850910663605 (0.3723824280546978) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3105 (0.32421418130397733) |
|
| > loader_time: 0.0328 (0.033851385302841666) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:44:17 -- STEP: 1330/1935 -- GLOBAL_STEP: 5200[0m |
|
| > loss_text_ce: 0.043980930000543594 (0.04404743741777612) |
|
| > loss_mel_ce: 2.83390212059021 (2.9353286943937604) |
|
| > loss: 0.359735369682312 (0.37242201636160216) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2958 (0.32421852663943584) |
|
| > loader_time: 0.0337 (0.033885537592092) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:44:43 -- STEP: 1380/1935 -- GLOBAL_STEP: 5250[0m |
|
| > loss_text_ce: 0.043862562626600266 (0.044048326600181056) |
|
| > loss_mel_ce: 2.873419761657715 (2.935377692830735) |
|
| > loss: 0.3646602928638458 (0.3724282522996267) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3433 (0.32429848598397265) |
|
| > loader_time: 0.0355 (0.033888459378394575) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:45:09 -- STEP: 1430/1935 -- GLOBAL_STEP: 5300[0m |
|
| > loss_text_ce: 0.04354468733072281 (0.0440443392035428) |
|
| > loss_mel_ce: 2.945492744445801 (2.9356061176820223) |
|
| > loss: 0.37362968921661377 (0.37245630701105087) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2938 (0.3242999908807393) |
|
| > loader_time: 0.0324 (0.03389305068062742) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:45:36 -- STEP: 1480/1935 -- GLOBAL_STEP: 5350[0m |
|
| > loss_text_ce: 0.04311772808432579 (0.04404592641906164) |
|
| > loss_mel_ce: 2.894596576690674 (2.9356674147618773) |
|
| > loss: 0.36721429228782654 (0.372464167588466) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2888 (0.3244998029760411) |
|
| > loader_time: 0.0341 (0.033929892166240865) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:46:01 -- STEP: 1530/1935 -- GLOBAL_STEP: 5400[0m |
|
| > loss_text_ce: 0.046486467123031616 (0.04404534756212069) |
|
| > loss_mel_ce: 2.9724786281585693 (2.935749046007791) |
|
| > loss: 0.3773706257343292 (0.3724742991277596) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3543 (0.32439033657896754) |
|
| > loader_time: 0.0351 (0.03395629116133156) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:46:28 -- STEP: 1580/1935 -- GLOBAL_STEP: 5450[0m |
|
| > loss_text_ce: 0.04366597160696983 (0.04403959689072421) |
|
| > loss_mel_ce: 2.9990522861480713 (2.935877625248099) |
|
| > loss: 0.38033977150917053 (0.3724896527732476) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3371 (0.3245646912840347) |
|
| > loader_time: 0.0361 (0.033997958823095455) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:46:54 -- STEP: 1630/1935 -- GLOBAL_STEP: 5500[0m |
|
| > loss_text_ce: 0.04664118215441704 (0.04405204300637265) |
|
| > loss_mel_ce: 2.97408390045166 (2.935852990998811) |
|
| > loss: 0.3775906264781952 (0.3724881292486484) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3172 (0.3244370144569068) |
|
| > loader_time: 0.0349 (0.034011986358034166) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:47:19 -- STEP: 1680/1935 -- GLOBAL_STEP: 5550[0m |
|
| > loss_text_ce: 0.04319654032588005 (0.04404604550051907) |
|
| > loss_mel_ce: 2.985860586166382 (2.935648836266426) |
|
| > loss: 0.378632128238678 (0.37246186020118854) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3135 (0.324268368028459) |
|
| > loader_time: 0.035 (0.03403290666285021) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:47:46 -- STEP: 1730/1935 -- GLOBAL_STEP: 5600[0m |
|
| > loss_text_ce: 0.04323196783661842 (0.04404535401677125) |
|
| > loss_mel_ce: 2.893714427947998 (2.9358584262043057) |
|
| > loss: 0.36711829900741577 (0.37248797253032645) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3412 (0.32424892896861685) |
|
| > loader_time: 0.0356 (0.03402680055254462) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:48:12 -- STEP: 1780/1935 -- GLOBAL_STEP: 5650[0m |
|
| > loss_text_ce: 0.04630498215556145 (0.04404018878560058) |
|
| > loss_mel_ce: 2.965853214263916 (2.935594178719466) |
|
| > loss: 0.3765197694301605 (0.3724542959687416) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3466 (0.3243212959739598) |
|
| > loader_time: 0.0356 (0.03403891592883) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:48:38 -- STEP: 1830/1935 -- GLOBAL_STEP: 5700[0m |
|
| > loss_text_ce: 0.04581262543797493 (0.04404034027473527) |
|
| > loss_mel_ce: 3.021080732345581 (2.9354123442550817) |
|
| > loss: 0.38336166739463806 (0.37243158556724515) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2976 (0.32429997269573096) |
|
| > loader_time: 0.0356 (0.03406984558522382) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:49:04 -- STEP: 1880/1935 -- GLOBAL_STEP: 5750[0m |
|
| > loss_text_ce: 0.046157773584127426 (0.04403561079042394) |
|
| > loss_mel_ce: 2.9095065593719482 (2.9351444704735523) |
|
| > loss: 0.36945804953575134 (0.3723975101842526) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3254 (0.3244845276183269) |
|
| > loader_time: 0.0342 (0.03410252751188083) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:49:30 -- STEP: 1930/1935 -- GLOBAL_STEP: 5800[0m |
|
| > loss_text_ce: 0.044207923114299774 (0.044031247568022444) |
|
| > loss_mel_ce: 2.960310697555542 (2.9352133633559223) |
|
| > loss: 0.3755648136138916 (0.3724055763971003) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3468 (0.32453110131575025) |
|
| > loader_time: 0.0343 (0.034032248833018953) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[92m 0.11520640296165388 [0m(-0.0008922008552936511) |
|
| > avg_loss_text_ce:[92m 0.04217620246639153 [0m(-0.0002854715543564104) |
|
| > avg_loss_mel_ce:[92m 2.9906537653219822 [0m(-0.005137441134211773) |
|
| > avg_loss:[92m 3.0328299674120807 [0m(-0.0054229076462584125) |
|
|
|
|
|
[4m[1m > EPOCH: 3/10[0m |
|
--> /checkpoints/GPT_XTTS_FT-December-23-2024_04+57PM-0000000 |
|
|
|
[1m > TRAINING (2024-12-23 17:50:21) [0m |
|
|
|
[1m --> TIME: 2024-12-23 17:50:45 -- STEP: 45/1935 -- GLOBAL_STEP: 5850[0m |
|
| > loss_text_ce: 0.04389598220586777 (0.04419783072339163) |
|
| > loss_mel_ce: 2.9726803302764893 (2.9488741132948135) |
|
| > loss: 0.3770720362663269 (0.3741339928574034) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2918 (0.3255475309160021) |
|
| > loader_time: 0.0323 (0.033482694625854494) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:51:11 -- STEP: 95/1935 -- GLOBAL_STEP: 5900[0m |
|
| > loss_text_ce: 0.04431807994842529 (0.04418157429287309) |
|
| > loss_mel_ce: 2.9853789806365967 (2.943947952672055) |
|
| > loss: 0.37871211767196655 (0.3735161916205759) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3673 (0.32476672875253776) |
|
| > loader_time: 0.0342 (0.03398661362497431) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:51:37 -- STEP: 145/1935 -- GLOBAL_STEP: 5950[0m |
|
| > loss_text_ce: 0.043669190257787704 (0.04405144699688616) |
|
| > loss_mel_ce: 2.957082748413086 (2.9428931828202876) |
|
| > loss: 0.3750939965248108 (0.37336807970342967) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3133 (0.3231423065580171) |
|
| > loader_time: 0.0331 (0.033860216469600274) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:52:04 -- STEP: 195/1935 -- GLOBAL_STEP: 6000[0m |
|
| > loss_text_ce: 0.0440300889313221 (0.04397747944562863) |
|
| > loss_mel_ce: 2.9671554565429688 (2.940320914830917) |
|
| > loss: 0.3763982057571411 (0.37303730004873037) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.383 (0.3257982241801727) |
|
| > loader_time: 0.0353 (0.03394078719310271) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:52:30 -- STEP: 245/1935 -- GLOBAL_STEP: 6050[0m |
|
| > loss_text_ce: 0.043720509856939316 (0.04395488302926628) |
|
| > loss_mel_ce: 2.9389169216156006 (2.9375994439027746) |
|
| > loss: 0.3728296756744385 (0.3726942912656434) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3118 (0.3244649410247803) |
|
| > loader_time: 0.0329 (0.0339020758259053) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:52:55 -- STEP: 295/1935 -- GLOBAL_STEP: 6100[0m |
|
| > loss_text_ce: 0.042607445269823074 (0.04390461774448217) |
|
| > loss_mel_ce: 2.9363276958465576 (2.9377246517246043) |
|
| > loss: 0.37236690521240234 (0.37270365902933034) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3304 (0.3241666947380972) |
|
| > loader_time: 0.0329 (0.03385543823242187) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:53:21 -- STEP: 345/1935 -- GLOBAL_STEP: 6150[0m |
|
| > loss_text_ce: 0.04517971724271774 (0.04393100734206214) |
|
| > loss_mel_ce: 2.9233200550079346 (2.9378671231477163) |
|
| > loss: 0.3710624575614929 (0.3727247664893886) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3455 (0.32398574525031476) |
|
| > loader_time: 0.0332 (0.03390110886615255) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:53:47 -- STEP: 395/1935 -- GLOBAL_STEP: 6200[0m |
|
| > loss_text_ce: 0.04473860189318657 (0.043957597205910505) |
|
| > loss_mel_ce: 2.890948534011841 (2.9365100051783313) |
|
| > loss: 0.36696088314056396 (0.3725584504725061) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3067 (0.32356437489956225) |
|
| > loader_time: 0.0332 (0.033925327470030936) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:54:13 -- STEP: 445/1935 -- GLOBAL_STEP: 6250[0m |
|
| > loss_text_ce: 0.044554971158504486 (0.0439584340523468) |
|
| > loss_mel_ce: 2.8203110694885254 (2.9362706746947906) |
|
| > loss: 0.358108252286911 (0.37252863886651055) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3206 (0.3235932794849523) |
|
| > loader_time: 0.0337 (0.03402527584118784) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:54:39 -- STEP: 495/1935 -- GLOBAL_STEP: 6300[0m |
|
| > loss_text_ce: 0.044815514236688614 (0.043948072624025926) |
|
| > loss_mel_ce: 2.8848485946655273 (2.935504487548212) |
|
| > loss: 0.366208016872406 (0.3724315702313127) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3357 (0.32321277868868115) |
|
| > loader_time: 0.0346 (0.03400754880423492) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:55:05 -- STEP: 545/1935 -- GLOBAL_STEP: 6350[0m |
|
| > loss_text_ce: 0.044077515602111816 (0.043934914333011034) |
|
| > loss_mel_ce: 3.0107295513153076 (2.9341784770335626) |
|
| > loss: 0.3818508982658386 (0.3722641740917068) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3151 (0.32306613397160794) |
|
| > loader_time: 0.0337 (0.03401753268110638) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:55:32 -- STEP: 595/1935 -- GLOBAL_STEP: 6400[0m |
|
| > loss_text_ce: 0.04499690607190132 (0.043890225812166665) |
|
| > loss_mel_ce: 2.918382167816162 (2.9343288185215783) |
|
| > loss: 0.3704223930835724 (0.37227738058867593) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2944 (0.3237159965418968) |
|
| > loader_time: 0.0333 (0.03404475540674029) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:55:57 -- STEP: 645/1935 -- GLOBAL_STEP: 6450[0m |
|
| > loss_text_ce: 0.04457513615489006 (0.04387362609769023) |
|
| > loss_mel_ce: 2.926753282546997 (2.9335772673288956) |
|
| > loss: 0.3714160621166229 (0.3721813617750658) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2961 (0.3232116181721059) |
|
| > loader_time: 0.0337 (0.03405808441398676) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:56:23 -- STEP: 695/1935 -- GLOBAL_STEP: 6500[0m |
|
| > loss_text_ce: 0.04136190190911293 (0.04386182135815241) |
|
| > loss_mel_ce: 2.8473315238952637 (2.9329244013312907) |
|
| > loss: 0.3610866665840149 (0.37209827797018385) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.309 (0.323205483731606) |
|
| > loader_time: 0.0338 (0.03400274763862004) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:56:49 -- STEP: 745/1935 -- GLOBAL_STEP: 6550[0m |
|
| > loss_text_ce: 0.04390119016170502 (0.043854928361689464) |
|
| > loss_mel_ce: 2.963810443878174 (2.9334867311004023) |
|
| > loss: 0.37596395611763 (0.37216770740963484) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3356 (0.32313548894536565) |
|
| > loader_time: 0.0344 (0.03395557339559463) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:57:15 -- STEP: 795/1935 -- GLOBAL_STEP: 6600[0m |
|
| > loss_text_ce: 0.0452137216925621 (0.04383724300928834) |
|
| > loss_mel_ce: 2.8940467834472656 (2.9327956070690036) |
|
| > loss: 0.36740756034851074 (0.3720791062469004) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2976 (0.3227830089113246) |
|
| > loader_time: 0.0323 (0.03393999975432388) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:57:41 -- STEP: 845/1935 -- GLOBAL_STEP: 6650[0m |
|
| > loss_text_ce: 0.04232422262430191 (0.04381974600475919) |
|
| > loss_mel_ce: 2.980522871017456 (2.9328051183350694) |
|
| > loss: 0.37785589694976807 (0.3720781080116183) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.347 (0.3235216736088136) |
|
| > loader_time: 0.0218 (0.03394351626288959) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:58:07 -- STEP: 895/1935 -- GLOBAL_STEP: 6700[0m |
|
| > loss_text_ce: 0.041043851524591446 (0.043803848596758004) |
|
| > loss_mel_ce: 2.9321210384368896 (2.9321229852111648) |
|
| > loss: 0.37164559960365295 (0.3719908542140237) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3622 (0.3237882909828057) |
|
| > loader_time: 0.0176 (0.033313565813629285) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:58:32 -- STEP: 945/1935 -- GLOBAL_STEP: 6750[0m |
|
| > loss_text_ce: 0.043374352157115936 (0.04379329626206997) |
|
| > loss_mel_ce: 2.8657596111297607 (2.9318714124185066) |
|
| > loss: 0.36364173889160156 (0.37195808865405905) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3141 (0.3238470973161161) |
|
| > loader_time: 0.021 (0.032620722654635274) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:58:58 -- STEP: 995/1935 -- GLOBAL_STEP: 6800[0m |
|
| > loss_text_ce: 0.04423433169722557 (0.043788572310952074) |
|
| > loss_mel_ce: 2.959885835647583 (2.932079386591312) |
|
| > loss: 0.37551501393318176 (0.3719834949502994) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3498 (0.3235413985036724) |
|
| > loader_time: 0.0352 (0.03225420659510928) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:59:24 -- STEP: 1045/1935 -- GLOBAL_STEP: 6850[0m |
|
| > loss_text_ce: 0.04572456330060959 (0.043796327367924034) |
|
| > loss_mel_ce: 2.9105165004730225 (2.9320155015972817) |
|
| > loss: 0.3695301413536072 (0.3719764786759063) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.344 (0.32356065289255526) |
|
| > loader_time: 0.0345 (0.03235990944091208) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 17:59:50 -- STEP: 1095/1935 -- GLOBAL_STEP: 6900[0m |
|
| > loss_text_ce: 0.04462014511227608 (0.043805606693845894) |
|
| > loss_mel_ce: 2.9592063426971436 (2.931904560681348) |
|
| > loss: 0.3754782974720001 (0.3719637708849015) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.319 (0.32343304668931644) |
|
| > loader_time: 0.0335 (0.03243091901143387) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:00:16 -- STEP: 1145/1935 -- GLOBAL_STEP: 6950[0m |
|
| > loss_text_ce: 0.04197946563363075 (0.043799201987976585) |
|
| > loss_mel_ce: 2.9674367904663086 (2.931973241822688) |
|
| > loss: 0.376177042722702 (0.37197155546413246) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3203 (0.32345597691931555) |
|
| > loader_time: 0.0353 (0.032518479501316096) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:00:42 -- STEP: 1195/1935 -- GLOBAL_STEP: 7000[0m |
|
| > loss_text_ce: 0.04529254510998726 (0.04379010642859226) |
|
| > loss_mel_ce: 2.8434596061706543 (2.9318047345931566) |
|
| > loss: 0.36109402775764465 (0.37194935509849275) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3142 (0.3235284236684501) |
|
| > loader_time: 0.0372 (0.032604663441869484) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:01:08 -- STEP: 1245/1935 -- GLOBAL_STEP: 7050[0m |
|
| > loss_text_ce: 0.04367287829518318 (0.043776580748189395) |
|
| > loss_mel_ce: 2.9795727729797363 (2.9320282413298817) |
|
| > loss: 0.3779056966304779 (0.3719756027302109) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.314 (0.3235317218734556) |
|
| > loader_time: 0.034 (0.03269115356077628) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:01:34 -- STEP: 1295/1935 -- GLOBAL_STEP: 7100[0m |
|
| > loss_text_ce: 0.043927062302827835 (0.043782553539888275) |
|
| > loss_mel_ce: 2.914491891860962 (2.9322439420177173) |
|
| > loss: 0.3698023557662964 (0.372003311908383) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3364 (0.3237539291381835) |
|
| > loader_time: 0.0341 (0.03280218319543076) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:02:00 -- STEP: 1345/1935 -- GLOBAL_STEP: 7150[0m |
|
| > loss_text_ce: 0.04412569850683212 (0.04378321324804016) |
|
| > loss_mel_ce: 2.8944873809814453 (2.9317375872658054) |
|
| > loss: 0.36732664704322815 (0.3719401000822341) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3172 (0.3238001151598961) |
|
| > loader_time: 0.0339 (0.032873140391807075) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:02:26 -- STEP: 1395/1935 -- GLOBAL_STEP: 7200[0m |
|
| > loss_text_ce: 0.04310794919729233 (0.043782601684438746) |
|
| > loss_mel_ce: 2.803781509399414 (2.9316740642738983) |
|
| > loss: 0.35586118698120117 (0.3719320832614828) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3408 (0.32371161710404134) |
|
| > loader_time: 0.035 (0.03293051224028335) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:02:52 -- STEP: 1445/1935 -- GLOBAL_STEP: 7250[0m |
|
| > loss_text_ce: 0.04416247084736824 (0.04378865334917519) |
|
| > loss_mel_ce: 2.844228982925415 (2.9320471394020755) |
|
| > loss: 0.36104893684387207 (0.37197947413451093) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.298 (0.3237154490394986) |
|
| > loader_time: 0.034 (0.03297152387229626) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:03:19 -- STEP: 1495/1935 -- GLOBAL_STEP: 7300[0m |
|
| > loss_text_ce: 0.044087544083595276 (0.04378155086899681) |
|
| > loss_mel_ce: 2.978963851928711 (2.932261862643184) |
|
| > loss: 0.3778814375400543 (0.37200542668435066) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3308 (0.3237749267182621) |
|
| > loader_time: 0.0337 (0.03303555357815024) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:03:44 -- STEP: 1545/1935 -- GLOBAL_STEP: 7350[0m |
|
| > loss_text_ce: 0.04416324570775032 (0.043785130507448346) |
|
| > loss_mel_ce: 2.845491409301758 (2.932013905009788) |
|
| > loss: 0.3612068295478821 (0.37197487946467084) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3383 (0.3235501494608263) |
|
| > loader_time: 0.0342 (0.03310500178908061) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:04:10 -- STEP: 1595/1935 -- GLOBAL_STEP: 7400[0m |
|
| > loss_text_ce: 0.043360862880945206 (0.04378603541038253) |
|
| > loss_mel_ce: 2.9093871116638184 (2.931788141376169) |
|
| > loss: 0.36909350752830505 (0.3719467721949548) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3199 (0.32353442096411217) |
|
| > loader_time: 0.0355 (0.03315326920871069) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:04:36 -- STEP: 1645/1935 -- GLOBAL_STEP: 7450[0m |
|
| > loss_text_ce: 0.044811904430389404 (0.0437784491542806) |
|
| > loss_mel_ce: 2.8362576961517334 (2.9316471876706727) |
|
| > loss: 0.36013370752334595 (0.37192820468209775) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3427 (0.32355444148681056) |
|
| > loader_time: 0.0359 (0.03320819367753688) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:05:03 -- STEP: 1695/1935 -- GLOBAL_STEP: 7500[0m |
|
| > loss_text_ce: 0.04182503744959831 (0.04377775807428149) |
|
| > loss_mel_ce: 2.8973422050476074 (2.9317372251752523) |
|
| > loss: 0.36739590764045715 (0.3719393729987747) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3646 (0.32366159405328504) |
|
| > loader_time: 0.0352 (0.03324615245020133) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:05:29 -- STEP: 1745/1935 -- GLOBAL_STEP: 7550[0m |
|
| > loss_text_ce: 0.04325196146965027 (0.04377462461932341) |
|
| > loss_mel_ce: 2.901015281677246 (2.931799004617595) |
|
| > loss: 0.36803340911865234 (0.37194670380698897) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2973 (0.32375984068927904) |
|
| > loader_time: 0.035 (0.0333022435961616) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:05:55 -- STEP: 1795/1935 -- GLOBAL_STEP: 7600[0m |
|
| > loss_text_ce: 0.042820170521736145 (0.0437700807010563) |
|
| > loss_mel_ce: 2.9464900493621826 (2.931675157573559) |
|
| > loss: 0.3736637830734253 (0.37193065488238836) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3503 (0.3237879166031944) |
|
| > loader_time: 0.0361 (0.03335196606628071) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:06:21 -- STEP: 1845/1935 -- GLOBAL_STEP: 7650[0m |
|
| > loss_text_ce: 0.04360957816243172 (0.04376144919413215) |
|
| > loss_mel_ce: 2.9469339847564697 (2.931882491925865) |
|
| > loss: 0.37381795048713684 (0.3719554927452468) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3128 (0.3238445999176519) |
|
| > loader_time: 0.0343 (0.03338176926300126) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:06:48 -- STEP: 1895/1935 -- GLOBAL_STEP: 7700[0m |
|
| > loss_text_ce: 0.04363440349698067 (0.04375384587175299) |
|
| > loss_mel_ce: 2.9303979873657227 (2.9315652627114543) |
|
| > loss: 0.3717540502548218 (0.3719148887021559) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2925 (0.3239364825326721) |
|
| > loader_time: 0.0331 (0.033438865238884154) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[91m 0.11785494558738925 [0m(+0.00264854262573537) |
|
| > avg_loss_text_ce:[92m 0.04189789233108362 [0m(-0.00027831013530791454) |
|
| > avg_loss_mel_ce:[92m 2.9856211729723987 [0m(-0.005032592349583531) |
|
| > avg_loss:[92m 3.027519064720232 [0m(-0.005310902691848529) |
|
|
|
|
|
[4m[1m > EPOCH: 4/10[0m |
|
--> /checkpoints/GPT_XTTS_FT-December-23-2024_04+57PM-0000000 |
|
|
|
[1m > TRAINING (2024-12-23 18:07:57) [0m |
|
|
|
[1m --> TIME: 2024-12-23 18:08:03 -- STEP: 10/1935 -- GLOBAL_STEP: 7750[0m |
|
| > loss_text_ce: 0.045941274613142014 (0.04409577623009682) |
|
| > loss_mel_ce: 2.943263530731201 (2.93230345249176) |
|
| > loss: 0.37365061044692993 (0.37204990684986117) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3159 (0.3175251245498657) |
|
| > loader_time: 0.0232 (0.029979562759399413) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:08:29 -- STEP: 60/1935 -- GLOBAL_STEP: 7800[0m |
|
| > loss_text_ce: 0.043694622814655304 (0.04334079083055258) |
|
| > loss_mel_ce: 2.9411284923553467 (2.9209078033765157) |
|
| > loss: 0.37310290336608887 (0.3705310751994451) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3068 (0.32478661934534714) |
|
| > loader_time: 0.0311 (0.033213337262471526) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:08:55 -- STEP: 110/1935 -- GLOBAL_STEP: 7850[0m |
|
| > loss_text_ce: 0.04211580008268356 (0.04343741221184082) |
|
| > loss_mel_ce: 3.0386250019073486 (2.925362619486722) |
|
| > loss: 0.3850925862789154 (0.3711000046946786) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3484 (0.3227717399597168) |
|
| > loader_time: 0.0331 (0.03329174301841043) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:09:21 -- STEP: 160/1935 -- GLOBAL_STEP: 7900[0m |
|
| > loss_text_ce: 0.043942831456661224 (0.04342519324272871) |
|
| > loss_mel_ce: 2.9385552406311035 (2.924299047887326) |
|
| > loss: 0.37281227111816406 (0.3709655305370689) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2567 (0.3235163912177085) |
|
| > loader_time: 0.0346 (0.033486570417881036) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:09:47 -- STEP: 210/1935 -- GLOBAL_STEP: 7950[0m |
|
| > loss_text_ce: 0.043455854058265686 (0.0434941576172908) |
|
| > loss_mel_ce: 2.84953236579895 (2.927186306317648) |
|
| > loss: 0.36162352561950684 (0.3713350582690466) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3209 (0.32382268224443705) |
|
| > loader_time: 0.0339 (0.03351699965340751) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:10:13 -- STEP: 260/1935 -- GLOBAL_STEP: 8000[0m |
|
| > loss_text_ce: 0.04414989426732063 (0.043432762574117906) |
|
| > loss_mel_ce: 2.8710813522338867 (2.9254207198436446) |
|
| > loss: 0.3644039034843445 (0.37110668558340804) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3393 (0.32431148015535793) |
|
| > loader_time: 0.034 (0.03358907791284413) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:10:39 -- STEP: 310/1935 -- GLOBAL_STEP: 8050[0m |
|
| > loss_text_ce: 0.042980633676052094 (0.04344666268556347) |
|
| > loss_mel_ce: 2.929654359817505 (2.926411726397853) |
|
| > loss: 0.3715793788433075 (0.37123229907405) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3216 (0.32449756822278414) |
|
| > loader_time: 0.033 (0.03366400657161587) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:11:05 -- STEP: 360/1935 -- GLOBAL_STEP: 8100[0m |
|
| > loss_text_ce: 0.0442245751619339 (0.04347111125373178) |
|
| > loss_mel_ce: 2.940523862838745 (2.9261507802539404) |
|
| > loss: 0.37309354543685913 (0.37120273676183496) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2732 (0.3243456370300715) |
|
| > loader_time: 0.0325 (0.03370239006148442) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:11:32 -- STEP: 410/1935 -- GLOBAL_STEP: 8150[0m |
|
| > loss_text_ce: 0.04346323758363724 (0.04347773924833391) |
|
| > loss_mel_ce: 2.915290355682373 (2.9246569935868427) |
|
| > loss: 0.3698441982269287 (0.371016841836092) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3443 (0.32473414758356606) |
|
| > loader_time: 0.0347 (0.0337363167506892) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:11:58 -- STEP: 460/1935 -- GLOBAL_STEP: 8200[0m |
|
| > loss_text_ce: 0.04258241131901741 (0.043495184758111184) |
|
| > loss_mel_ce: 2.8498542308807373 (2.9228433526080586) |
|
| > loss: 0.3615545928478241 (0.3707923174552296) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.298 (0.32481449324151745) |
|
| > loader_time: 0.0344 (0.03374129170956817) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:12:24 -- STEP: 510/1935 -- GLOBAL_STEP: 8250[0m |
|
| > loss_text_ce: 0.044135063886642456 (0.04351793851192091) |
|
| > loss_mel_ce: 2.972403049468994 (2.923172426223755) |
|
| > loss: 0.3770672678947449 (0.37083629621010206) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3472 (0.3252340125102623) |
|
| > loader_time: 0.0351 (0.033842141488019124) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:12:51 -- STEP: 560/1935 -- GLOBAL_STEP: 8300[0m |
|
| > loss_text_ce: 0.04394000023603439 (0.043514997399012954) |
|
| > loss_mel_ce: 3.059410572052002 (2.923877138325146) |
|
| > loss: 0.3879188299179077 (0.3709240176315819) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3266 (0.3255958591188704) |
|
| > loader_time: 0.0359 (0.033926765407834694) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:13:17 -- STEP: 610/1935 -- GLOBAL_STEP: 8350[0m |
|
| > loss_text_ce: 0.04442471265792847 (0.043512164795252134) |
|
| > loss_mel_ce: 2.951786994934082 (2.924008721601769) |
|
| > loss: 0.3745264708995819 (0.37094011145537037) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3374 (0.3257320787085859) |
|
| > loader_time: 0.0341 (0.03398225424719633) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:13:43 -- STEP: 660/1935 -- GLOBAL_STEP: 8400[0m |
|
| > loss_text_ce: 0.04169100522994995 (0.04350138415554255) |
|
| > loss_mel_ce: 2.8569047451019287 (2.923930871847906) |
|
| > loss: 0.36232447624206543 (0.3709290326093185) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.321 (0.3255685394460502) |
|
| > loader_time: 0.0345 (0.03402682976289225) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:14:09 -- STEP: 710/1935 -- GLOBAL_STEP: 8450[0m |
|
| > loss_text_ce: 0.042564552277326584 (0.043483995494078574) |
|
| > loss_mel_ce: 2.9646079540252686 (2.9235901160978948) |
|
| > loss: 0.3758965730667114 (0.3708842645648502) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3139 (0.32544641561911125) |
|
| > loader_time: 0.0359 (0.03399830401783255) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:14:35 -- STEP: 760/1935 -- GLOBAL_STEP: 8500[0m |
|
| > loss_text_ce: 0.04301004484295845 (0.04347135221683664) |
|
| > loss_mel_ce: 2.9400219917297363 (2.9228708998153095) |
|
| > loss: 0.3728789985179901 (0.3707927822282442) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3166 (0.32573061052121594) |
|
| > loader_time: 0.0338 (0.03401295260379185) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:15:02 -- STEP: 810/1935 -- GLOBAL_STEP: 8550[0m |
|
| > loss_text_ce: 0.04484014958143234 (0.04348341824003942) |
|
| > loss_mel_ce: 2.8583109378814697 (2.9225835432241003) |
|
| > loss: 0.36289387941360474 (0.37075837081597196) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3108 (0.3260758211583264) |
|
| > loader_time: 0.0347 (0.03405024475521504) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:15:28 -- STEP: 860/1935 -- GLOBAL_STEP: 8600[0m |
|
| > loss_text_ce: 0.043707408010959625 (0.043482154157272586) |
|
| > loss_mel_ce: 3.0158355236053467 (2.9229065260221803) |
|
| > loss: 0.3824428617954254 (0.3707985856158791) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3337 (0.32578532834385693) |
|
| > loader_time: 0.0335 (0.034048696728639735) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:15:54 -- STEP: 910/1935 -- GLOBAL_STEP: 8650[0m |
|
| > loss_text_ce: 0.04448193311691284 (0.04347390186573776) |
|
| > loss_mel_ce: 2.9670169353485107 (2.9231224749114495) |
|
| > loss: 0.37643736600875854 (0.3708245475868604) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3401 (0.3257870841812299) |
|
| > loader_time: 0.0406 (0.03410644269251553) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:16:20 -- STEP: 960/1935 -- GLOBAL_STEP: 8700[0m |
|
| > loss_text_ce: 0.04427425190806389 (0.04348534411983563) |
|
| > loss_mel_ce: 2.972329616546631 (2.9225035036603617) |
|
| > loss: 0.3770754933357239 (0.37074860654150465) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2951 (0.3257324357827502) |
|
| > loader_time: 0.0349 (0.03412884225447965) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:16:46 -- STEP: 1010/1935 -- GLOBAL_STEP: 8750[0m |
|
| > loss_text_ce: 0.04219009354710579 (0.04349617705413019) |
|
| > loss_mel_ce: 2.866166114807129 (2.922378874061132) |
|
| > loss: 0.3635445237159729 (0.3707343820003001) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3338 (0.32562442510434875) |
|
| > loader_time: 0.0347 (0.03408002428489149) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:17:12 -- STEP: 1060/1935 -- GLOBAL_STEP: 8800[0m |
|
| > loss_text_ce: 0.042585790157318115 (0.043489870372808194) |
|
| > loss_mel_ce: 2.8298165798187256 (2.922632525776918) |
|
| > loss: 0.35905030369758606 (0.37076530015131226) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3382 (0.32570571179659835) |
|
| > loader_time: 0.0347 (0.034078278631534155) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:17:38 -- STEP: 1110/1935 -- GLOBAL_STEP: 8850[0m |
|
| > loss_text_ce: 0.04229193553328514 (0.0434820430504309) |
|
| > loss_mel_ce: 2.927522659301758 (2.9223660954484005) |
|
| > loss: 0.37122681736946106 (0.37073101785268864) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2989 (0.3255057676418404) |
|
| > loader_time: 0.0356 (0.03404410809009993) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:18:04 -- STEP: 1160/1935 -- GLOBAL_STEP: 8900[0m |
|
| > loss_text_ce: 0.044701240956783295 (0.04347435531110085) |
|
| > loss_mel_ce: 2.958345651626587 (2.922693351424975) |
|
| > loss: 0.37538087368011475 (0.37077096382091773) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3165 (0.3254242453081849) |
|
| > loader_time: 0.0436 (0.0340802030316714) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:18:30 -- STEP: 1210/1935 -- GLOBAL_STEP: 8950[0m |
|
| > loss_text_ce: 0.04279661923646927 (0.043470346431963715) |
|
| > loss_mel_ce: 2.8996188640594482 (2.9220669911912673) |
|
| > loss: 0.3678019344806671 (0.37069216772051894) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3138 (0.3254948813067976) |
|
| > loader_time: 0.0278 (0.034108091977016884) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:18:57 -- STEP: 1260/1935 -- GLOBAL_STEP: 9000[0m |
|
| > loss_text_ce: 0.04185082018375397 (0.043468260028887366) |
|
| > loss_mel_ce: 2.904921531677246 (2.92191843929745) |
|
| > loss: 0.36834654211997986 (0.3706733380045209) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3136 (0.3256880514205444) |
|
| > loader_time: 0.0293 (0.03408588076394691) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:19:22 -- STEP: 1310/1935 -- GLOBAL_STEP: 9050[0m |
|
| > loss_text_ce: 0.041538700461387634 (0.043460016293835106) |
|
| > loss_mel_ce: 2.8932316303253174 (2.922129238288822) |
|
| > loss: 0.3668462932109833 (0.37069865731337626) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2713 (0.3253898141948318) |
|
| > loader_time: 0.0325 (0.034058774336603655) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:19:49 -- STEP: 1360/1935 -- GLOBAL_STEP: 9100[0m |
|
| > loss_text_ce: 0.0406179316341877 (0.043451999637353106) |
|
| > loss_mel_ce: 3.006847858428955 (2.9219238842234896) |
|
| > loss: 0.3809332251548767 (0.37067198604345364) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3181 (0.325379710863618) |
|
| > loader_time: 0.0344 (0.03409229411798352) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:20:15 -- STEP: 1410/1935 -- GLOBAL_STEP: 9150[0m |
|
| > loss_text_ce: 0.044359855353832245 (0.04344065846439378) |
|
| > loss_mel_ce: 2.9515128135681152 (2.9221870060508146) |
|
| > loss: 0.3744840919971466 (0.37070345863805637) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3368 (0.3252454187853112) |
|
| > loader_time: 0.0353 (0.03411846296161611) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:20:41 -- STEP: 1460/1935 -- GLOBAL_STEP: 9200[0m |
|
| > loss_text_ce: 0.04293922334909439 (0.04343882102315153) |
|
| > loss_mel_ce: 2.973626136779785 (2.9220499798043145) |
|
| > loss: 0.37707066535949707 (0.37068610064787444) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3364 (0.3252669429125847) |
|
| > loader_time: 0.0359 (0.03413145966725795) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:21:07 -- STEP: 1510/1935 -- GLOBAL_STEP: 9250[0m |
|
| > loss_text_ce: 0.04382101073861122 (0.04343933771974995) |
|
| > loss_mel_ce: 2.943674325942993 (2.9218773142391496) |
|
| > loss: 0.37343692779541016 (0.37066458207092573) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3535 (0.32530392005743536) |
|
| > loader_time: 0.036 (0.03414761609589019) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:21:33 -- STEP: 1560/1935 -- GLOBAL_STEP: 9300[0m |
|
| > loss_text_ce: 0.042947378009557724 (0.04343727308397111) |
|
| > loss_mel_ce: 2.9232068061828613 (2.921735812914678) |
|
| > loss: 0.3707692623138428 (0.37064663634086303) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3189 (0.325106618954585) |
|
| > loader_time: 0.035 (0.03415177372785707) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:21:59 -- STEP: 1610/1935 -- GLOBAL_STEP: 9350[0m |
|
| > loss_text_ce: 0.045330073684453964 (0.04344053648597337) |
|
| > loss_mel_ce: 2.892113447189331 (2.9216121898674827) |
|
| > loss: 0.3671804368495941 (0.3706315913555788) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3358 (0.3250992933415477) |
|
| > loader_time: 0.0345 (0.03415782066605837) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:22:25 -- STEP: 1660/1935 -- GLOBAL_STEP: 9400[0m |
|
| > loss_text_ce: 0.04188258573412895 (0.04343522006354061) |
|
| > loss_mel_ce: 2.906614303588867 (2.921360266926778) |
|
| > loss: 0.36856210231781006 (0.3705994364547446) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3358 (0.32520744628216836) |
|
| > loader_time: 0.0354 (0.03415181033582563) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:22:51 -- STEP: 1710/1935 -- GLOBAL_STEP: 9450[0m |
|
| > loss_text_ce: 0.04556925967335701 (0.04343271367251875) |
|
| > loss_mel_ce: 2.9928340911865234 (2.92151813088802) |
|
| > loss: 0.379800409078598 (0.37061885614492773) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3137 (0.32506188370330974) |
|
| > loader_time: 0.0344 (0.03416605218809243) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:23:17 -- STEP: 1760/1935 -- GLOBAL_STEP: 9500[0m |
|
| > loss_text_ce: 0.04206385090947151 (0.043424446171742286) |
|
| > loss_mel_ce: 2.8796207904815674 (2.9215646375309348) |
|
| > loss: 0.3652105927467346 (0.37062363604253024) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3092 (0.3252272721041331) |
|
| > loader_time: 0.0347 (0.03418368900364089) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:23:43 -- STEP: 1810/1935 -- GLOBAL_STEP: 9550[0m |
|
| > loss_text_ce: 0.04234180226922035 (0.04342090946658213) |
|
| > loss_mel_ce: 2.92434024810791 (2.9217295966754313) |
|
| > loss: 0.37083524465560913 (0.370643813866937) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3481 (0.32510984378625) |
|
| > loader_time: 0.0352 (0.03419519419169553) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:24:10 -- STEP: 1860/1935 -- GLOBAL_STEP: 9600[0m |
|
| > loss_text_ce: 0.04175252839922905 (0.04342103219360756) |
|
| > loss_mel_ce: 2.8431754112243652 (2.9216047943279317) |
|
| > loss: 0.360615998506546 (0.37062822887974434) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3015 (0.3252067026271614) |
|
| > loader_time: 0.0337 (0.03420287345045353) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:24:36 -- STEP: 1910/1935 -- GLOBAL_STEP: 9650[0m |
|
| > loss_text_ce: 0.04448603838682175 (0.043419894098221334) |
|
| > loss_mel_ce: 2.8089210987091064 (2.921525651372541) |
|
| > loss: 0.3566758930683136 (0.3706181937019241) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2878 (0.3251874117326985) |
|
| > loader_time: 0.0347 (0.03416672788989478) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[92m 0.11732191870910953 [0m(-0.0005330268782797187) |
|
| > avg_loss_text_ce:[92m 0.04163555255291438 [0m(-0.0002623397781692377) |
|
| > avg_loss_mel_ce:[92m 2.9807295955792816 [0m(-0.004891577393117075) |
|
| > avg_loss:[92m 3.022365142600704 [0m(-0.005153922119528254) |
|
|
|
|
|
[4m[1m > EPOCH: 5/10[0m |
|
--> /checkpoints/GPT_XTTS_FT-December-23-2024_04+57PM-0000000 |
|
|
|
[1m > TRAINING (2024-12-23 18:25:37) [0m |
|
|
|
[1m --> TIME: 2024-12-23 18:25:51 -- STEP: 25/1935 -- GLOBAL_STEP: 9700[0m |
|
| > loss_text_ce: 0.04444317892193794 (0.0432612070441246) |
|
| > loss_mel_ce: 2.927600383758545 (2.8971458148956297) |
|
| > loss: 0.3715054392814636 (0.3675508773326874) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4895 (0.3234361553192139) |
|
| > loader_time: 0.0366 (0.0336666488647461) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:26:17 -- STEP: 75/1935 -- GLOBAL_STEP: 9750[0m |
|
| > loss_text_ce: 0.0432705357670784 (0.0432078761359056) |
|
| > loss_mel_ce: 2.9630014896392822 (2.916092519760132) |
|
| > loss: 0.3757840096950531 (0.36991254965464276) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3436 (0.3212654972076416) |
|
| > loader_time: 0.0233 (0.033758942286173516) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:26:43 -- STEP: 125/1935 -- GLOBAL_STEP: 9800[0m |
|
| > loss_text_ce: 0.04386110231280327 (0.04317410284280778) |
|
| > loss_mel_ce: 2.8758301734924316 (2.9163439865112304) |
|
| > loss: 0.3649614155292511 (0.3699397611618042) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.274 (0.32346473503112794) |
|
| > loader_time: 0.0334 (0.03402656936645509) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:27:09 -- STEP: 175/1935 -- GLOBAL_STEP: 9850[0m |
|
| > loss_text_ce: 0.04334893077611923 (0.043232464641332645) |
|
| > loss_mel_ce: 2.973829984664917 (2.913656715665546) |
|
| > loss: 0.377147376537323 (0.3696111478124344) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3777 (0.3251959773472378) |
|
| > loader_time: 0.0341 (0.0338328320639474) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:27:35 -- STEP: 225/1935 -- GLOBAL_STEP: 9900[0m |
|
| > loss_text_ce: 0.04358653351664543 (0.04323835864663126) |
|
| > loss_mel_ce: 2.924248218536377 (2.9163671588897717) |
|
| > loss: 0.37097933888435364 (0.369950689739651) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3178 (0.3230699941847062) |
|
| > loader_time: 0.0204 (0.03285507414076064) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:28:00 -- STEP: 275/1935 -- GLOBAL_STEP: 9950[0m |
|
| > loss_text_ce: 0.04233744367957115 (0.04323229306123475) |
|
| > loss_mel_ce: 2.892852544784546 (2.916403948176992) |
|
| > loss: 0.3668987452983856 (0.3699545298923145) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.296 (0.32382185502485816) |
|
| > loader_time: 0.033 (0.03136758370832966) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:28:27 -- STEP: 325/1935 -- GLOBAL_STEP: 10000[0m |
|
| > loss_text_ce: 0.04188022017478943 (0.0432893649660624) |
|
| > loss_mel_ce: 2.878812074661255 (2.9172867650252137) |
|
| > loss: 0.36508652567863464 (0.37007201598240785) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2988 (0.32412475072420566) |
|
| > loader_time: 0.033 (0.03174655620868392) |
|
|
|
|
|
> CHECKPOINT : /checkpoints/GPT_XTTS_FT-December-23-2024_04+57PM-0000000/checkpoint_10000.pth |
|
|
|
[1m --> TIME: 2024-12-23 18:28:56 -- STEP: 375/1935 -- GLOBAL_STEP: 10050[0m |
|
| > loss_text_ce: 0.04342421516776085 (0.04329764613509176) |
|
| > loss_mel_ce: 2.8257386684417725 (2.9178082167307546) |
|
| > loss: 0.35864534974098206 (0.37013823246955874) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3736 (0.3241704737345378) |
|
| > loader_time: 0.0218 (0.030097412745157887) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:29:22 -- STEP: 425/1935 -- GLOBAL_STEP: 10100[0m |
|
| > loss_text_ce: 0.04409493878483772 (0.04329103239318902) |
|
| > loss_mel_ce: 3.0429210662841797 (2.919137836905088) |
|
| > loss: 0.38587701320648193 (0.37030360824921554) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3453 (0.32464017643648024) |
|
| > loader_time: 0.0347 (0.02977584165685318) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:29:48 -- STEP: 475/1935 -- GLOBAL_STEP: 10150[0m |
|
| > loss_text_ce: 0.041303012520074844 (0.043290470477781784) |
|
| > loss_mel_ce: 2.8718674182891846 (2.9173038758729644) |
|
| > loss: 0.36414629220962524 (0.3700742929860165) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3149 (0.3247289507012617) |
|
| > loader_time: 0.0331 (0.030283796912745433) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:30:15 -- STEP: 525/1935 -- GLOBAL_STEP: 10200[0m |
|
| > loss_text_ce: 0.043538033962249756 (0.043294906233038215) |
|
| > loss_mel_ce: 2.7901618480682373 (2.9170874541146423) |
|
| > loss: 0.3542124927043915 (0.37004779486429124) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3137 (0.32492740994408004) |
|
| > loader_time: 0.035 (0.03070695332118444) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:30:41 -- STEP: 575/1935 -- GLOBAL_STEP: 10250[0m |
|
| > loss_text_ce: 0.04296332597732544 (0.043291709487852825) |
|
| > loss_mel_ce: 2.8468716144561768 (2.917071255808292) |
|
| > loss: 0.3612293601036072 (0.37004537043364144) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3399 (0.3247663837930428) |
|
| > loader_time: 0.0344 (0.031054873259171202) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:31:07 -- STEP: 625/1935 -- GLOBAL_STEP: 10300[0m |
|
| > loss_text_ce: 0.043891653418540955 (0.04329174993038176) |
|
| > loss_mel_ce: 2.8353042602539062 (2.9165918403625497) |
|
| > loss: 0.35989949107170105 (0.3699854485511779) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3369 (0.32433023643493636) |
|
| > loader_time: 0.0346 (0.031345357131958004) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:31:33 -- STEP: 675/1935 -- GLOBAL_STEP: 10350[0m |
|
| > loss_text_ce: 0.041904252022504807 (0.04328874516818257) |
|
| > loss_mel_ce: 2.8854849338531494 (2.9166894852673573) |
|
| > loss: 0.3659236431121826 (0.36999727871682886) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2985 (0.3242364696220113) |
|
| > loader_time: 0.0346 (0.031493577957153296) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:31:59 -- STEP: 725/1935 -- GLOBAL_STEP: 10400[0m |
|
| > loss_text_ce: 0.04336021840572357 (0.04327273707965321) |
|
| > loss_mel_ce: 2.912320375442505 (2.915719870534437) |
|
| > loss: 0.3694600760936737 (0.36987407602112843) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.336 (0.3242969802330279) |
|
| > loader_time: 0.0345 (0.03168250248349944) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:32:25 -- STEP: 775/1935 -- GLOBAL_STEP: 10450[0m |
|
| > loss_text_ce: 0.04330456629395485 (0.04328349051456294) |
|
| > loss_mel_ce: 2.968594789505005 (2.916257211623654) |
|
| > loss: 0.3764874339103699 (0.3699425878447869) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.328 (0.3244223606971003) |
|
| > loader_time: 0.0335 (0.03190211019208351) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:32:51 -- STEP: 825/1935 -- GLOBAL_STEP: 10500[0m |
|
| > loss_text_ce: 0.04258396103978157 (0.04328545018579017) |
|
| > loss_mel_ce: 2.9553189277648926 (2.9159052412437676) |
|
| > loss: 0.37473785877227783 (0.3698988365765772) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3104 (0.32451376886078803) |
|
| > loader_time: 0.0326 (0.032062315218376346) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:33:17 -- STEP: 875/1935 -- GLOBAL_STEP: 10550[0m |
|
| > loss_text_ce: 0.04283343255519867 (0.04325816035696435) |
|
| > loss_mel_ce: 2.9470701217651367 (2.915581841060094) |
|
| > loss: 0.3737379312515259 (0.3698550003596713) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3172 (0.32449568557739256) |
|
| > loader_time: 0.0337 (0.03215720966884063) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:33:43 -- STEP: 925/1935 -- GLOBAL_STEP: 10600[0m |
|
| > loss_text_ce: 0.04271556809544563 (0.04324949288287675) |
|
| > loss_mel_ce: 3.0016939640045166 (2.915428590259037) |
|
| > loss: 0.38055118918418884 (0.3698347605563497) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2944 (0.32456299214749723) |
|
| > loader_time: 0.0328 (0.03222357337539255) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:34:09 -- STEP: 975/1935 -- GLOBAL_STEP: 10650[0m |
|
| > loss_text_ce: 0.04257501661777496 (0.0432381328138021) |
|
| > loss_mel_ce: 2.8746912479400635 (2.915854767041329) |
|
| > loss: 0.36465829610824585 (0.3698866127393183) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3584 (0.3246348906786014) |
|
| > loader_time: 0.035 (0.032338268573467485) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:34:35 -- STEP: 1025/1935 -- GLOBAL_STEP: 10700[0m |
|
| > loss_text_ce: 0.04429507628083229 (0.04324214050682577) |
|
| > loss_mel_ce: 2.9656527042388916 (2.9151351747280216) |
|
| > loss: 0.3762434720993042 (0.3697971646669433) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3834 (0.3243814675400897) |
|
| > loader_time: 0.0349 (0.032409997567897836) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:35:01 -- STEP: 1075/1935 -- GLOBAL_STEP: 10750[0m |
|
| > loss_text_ce: 0.042120181024074554 (0.04323840504815409) |
|
| > loss_mel_ce: 3.038831949234009 (2.915596606454185) |
|
| > loss: 0.3851190209388733 (0.3698543766210244) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3368 (0.32421125434165776) |
|
| > loader_time: 0.0348 (0.032544512859610575) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:35:27 -- STEP: 1125/1935 -- GLOBAL_STEP: 10800[0m |
|
| > loss_text_ce: 0.044841740280389786 (0.04323435776101216) |
|
| > loss_mel_ce: 2.882383108139038 (2.9164097571902823) |
|
| > loss: 0.3659031093120575 (0.3699555145899453) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2956 (0.324010208129883) |
|
| > loader_time: 0.033 (0.03261446974012581) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:35:53 -- STEP: 1175/1935 -- GLOBAL_STEP: 10850[0m |
|
| > loss_text_ce: 0.042451005429029465 (0.04322359932229873) |
|
| > loss_mel_ce: 2.9083356857299805 (2.9169172323511017) |
|
| > loss: 0.3688483238220215 (0.37001760409233403) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.317 (0.3241261922552233) |
|
| > loader_time: 0.0277 (0.03270575990068145) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:36:19 -- STEP: 1225/1935 -- GLOBAL_STEP: 10900[0m |
|
| > loss_text_ce: 0.043183181434869766 (0.0432156985055427) |
|
| > loss_mel_ce: 2.9410622119903564 (2.9165168943210533) |
|
| > loss: 0.3730306625366211 (0.3699665742017784) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3181 (0.3241556909132978) |
|
| > loader_time: 0.0346 (0.03276506929981456) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:36:45 -- STEP: 1275/1935 -- GLOBAL_STEP: 10950[0m |
|
| > loss_text_ce: 0.04443821310997009 (0.04320994216729614) |
|
| > loss_mel_ce: 2.840867042541504 (2.91653276200388) |
|
| > loss: 0.36066314578056335 (0.3699678381284076) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3041 (0.32420332515940975) |
|
| > loader_time: 0.034 (0.03280345019172212) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:37:12 -- STEP: 1325/1935 -- GLOBAL_STEP: 11000[0m |
|
| > loss_text_ce: 0.042237505316734314 (0.043209927287304176) |
|
| > loss_mel_ce: 2.8880722522735596 (2.9161340265454005) |
|
| > loss: 0.3662887215614319 (0.3699179943777479) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2515 (0.32425347418155337) |
|
| > loader_time: 0.0331 (0.03286216807815258) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:37:38 -- STEP: 1375/1935 -- GLOBAL_STEP: 11050[0m |
|
| > loss_text_ce: 0.04416709393262863 (0.0432034368108619) |
|
| > loss_mel_ce: 2.885958671569824 (2.916076654780998) |
|
| > loss: 0.3662657141685486 (0.36991001155159675) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3307 (0.3244430354725235) |
|
| > loader_time: 0.0412 (0.03294136307456271) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:38:04 -- STEP: 1425/1935 -- GLOBAL_STEP: 11100[0m |
|
| > loss_text_ce: 0.04377652704715729 (0.04319407968144664) |
|
| > loss_mel_ce: 2.991353988647461 (2.9159391506931276) |
|
| > loss: 0.37939131259918213 (0.3698916539183832) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3186 (0.32445124140956993) |
|
| > loader_time: 0.0359 (0.03299469730310268) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:38:31 -- STEP: 1475/1935 -- GLOBAL_STEP: 11150[0m |
|
| > loss_text_ce: 0.04093214124441147 (0.04318454238822901) |
|
| > loss_mel_ce: 2.8834095001220703 (2.9158257439176953) |
|
| > loss: 0.3655427098274231 (0.3698762858722167) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3115 (0.32480418253753174) |
|
| > loader_time: 0.0356 (0.033058492692850355) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:38:57 -- STEP: 1525/1935 -- GLOBAL_STEP: 11200[0m |
|
| > loss_text_ce: 0.04222259297966957 (0.04317818211238889) |
|
| > loss_mel_ce: 2.917328119277954 (2.916142579219384) |
|
| > loss: 0.3699438273906708 (0.3699150952159379) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3469 (0.3247986157214059) |
|
| > loader_time: 0.0346 (0.03310664958641169) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:39:23 -- STEP: 1575/1935 -- GLOBAL_STEP: 11250[0m |
|
| > loss_text_ce: 0.042306091636419296 (0.04318386102243073) |
|
| > loss_mel_ce: 2.9170706272125244 (2.9164733034466974) |
|
| > loss: 0.36992210149765015 (0.36995714560387605) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3345 (0.32495242345900793) |
|
| > loader_time: 0.0342 (0.033157079030596906) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:39:50 -- STEP: 1625/1935 -- GLOBAL_STEP: 11300[0m |
|
| > loss_text_ce: 0.043161243200302124 (0.043166750300389044) |
|
| > loss_mel_ce: 2.955958604812622 (2.916499944540174) |
|
| > loss: 0.37488996982574463 (0.3699583368851586) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3167 (0.3250483727088344) |
|
| > loader_time: 0.0336 (0.03318087607163639) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:40:16 -- STEP: 1675/1935 -- GLOBAL_STEP: 11350[0m |
|
| > loss_text_ce: 0.044227343052625656 (0.043164180635071485) |
|
| > loss_mel_ce: 2.864532470703125 (2.9162159615132377) |
|
| > loss: 0.36359497904777527 (0.3699225177871645) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3467 (0.3250605067922111) |
|
| > loader_time: 0.0356 (0.03323287052894697) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:40:42 -- STEP: 1725/1935 -- GLOBAL_STEP: 11400[0m |
|
| > loss_text_ce: 0.04325392469763756 (0.04317121356941649) |
|
| > loss_mel_ce: 2.866262674331665 (2.9161477797964377) |
|
| > loss: 0.3636895716190338 (0.36991487420123537) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3424 (0.3249572172026706) |
|
| > loader_time: 0.0357 (0.03327176716016678) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:41:08 -- STEP: 1775/1935 -- GLOBAL_STEP: 11450[0m |
|
| > loss_text_ce: 0.04270066320896149 (0.04316391118185618) |
|
| > loss_mel_ce: 2.8696305751800537 (2.9163814993307655) |
|
| > loss: 0.36404141783714294 (0.3699431763232592) |
|
| > grad_norm: 0 (0.0) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.316 (0.3248676963591242) |
|
| > loader_time: 0.034 (0.033303988819390846) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:41:34 -- STEP: 1825/1935 -- GLOBAL_STEP: 11500[0m |
|
| > loss_text_ce: 0.043397754430770874 (0.04316130157815264) |
|
| > loss_mel_ce: 2.95011568069458 (2.9163296111642527) |
|
| > loss: 0.374189168214798 (0.3699363640889728) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3338 (0.32487217328319834) |
|
| > loader_time: 0.0351 (0.03334324823666914) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:42:00 -- STEP: 1875/1935 -- GLOBAL_STEP: 11550[0m |
|
| > loss_text_ce: 0.04149981960654259 (0.04315198429028191) |
|
| > loss_mel_ce: 2.916468381881714 (2.9160782572428414) |
|
| > loss: 0.36974602937698364 (0.3699037802219389) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3381 (0.32492728144327826) |
|
| > loader_time: 0.0333 (0.03333572413126616) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:42:26 -- STEP: 1925/1935 -- GLOBAL_STEP: 11600[0m |
|
| > loss_text_ce: 0.04343201965093613 (0.04315107348870919) |
|
| > loss_mel_ce: 2.923103094100952 (2.916058804648266) |
|
| > loss: 0.3708168864250183 (0.3699012348094542) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3492 (0.3247908003918539) |
|
| > loader_time: 0.0372 (0.03330376649831785) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[92m 0.116858827947366 [0m(-0.0004630907617435287) |
|
| > avg_loss_text_ce:[92m 0.041378771321791616 [0m(-0.00025678123112276435) |
|
| > avg_loss_mel_ce:[92m 2.9779540794064308 [0m(-0.002775516172850878) |
|
| > avg_loss:[92m 3.0193328496181597 [0m(-0.003032292982544149) |
|
|
|
> BEST MODEL : /checkpoints/GPT_XTTS_FT-December-23-2024_04+57PM-0000000/best_model_11610.pth |
|
|
|
[4m[1m > EPOCH: 6/10[0m |
|
--> /checkpoints/GPT_XTTS_FT-December-23-2024_04+57PM-0000000 |
|
|
|
[1m > TRAINING (2024-12-23 18:43:24) [0m |
|
|
|
[1m --> TIME: 2024-12-23 18:43:45 -- STEP: 40/1935 -- GLOBAL_STEP: 11650[0m |
|
| > loss_text_ce: 0.0417630560696125 (0.042912021093070504) |
|
| > loss_mel_ce: 2.9677395820617676 (2.9003801703453065) |
|
| > loss: 0.37618783116340637 (0.3679115243256092) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3254 (0.32396247386932375) |
|
| > loader_time: 0.0337 (0.022135043144226076) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:44:11 -- STEP: 90/1935 -- GLOBAL_STEP: 11700[0m |
|
| > loss_text_ce: 0.041786350309848785 (0.04307159355117213) |
|
| > loss_mel_ce: 2.8504979610443115 (2.9036411815219454) |
|
| > loss: 0.36153554916381836 (0.3683390974998474) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3096 (0.3228728930155435) |
|
| > loader_time: 0.0327 (0.027751533190409346) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:44:38 -- STEP: 140/1935 -- GLOBAL_STEP: 11750[0m |
|
| > loss_text_ce: 0.04488437622785568 (0.043083092011511316) |
|
| > loss_mel_ce: 3.009864568710327 (2.9076006531715395) |
|
| > loss: 0.381843626499176 (0.3688354696546282) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2768 (0.32580814020974286) |
|
| > loader_time: 0.0328 (0.029737230709620883) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:45:03 -- STEP: 190/1935 -- GLOBAL_STEP: 11800[0m |
|
| > loss_text_ce: 0.04272224381566048 (0.04316343218088147) |
|
| > loss_mel_ce: 2.8825104236602783 (2.910833728940862) |
|
| > loss: 0.3656540811061859 (0.36924964613036093) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3085 (0.32375300432506343) |
|
| > loader_time: 0.0327 (0.0306162018524973) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:45:30 -- STEP: 240/1935 -- GLOBAL_STEP: 11850[0m |
|
| > loss_text_ce: 0.04191041365265846 (0.04315170824217297) |
|
| > loss_mel_ce: 2.9714043140411377 (2.912479610244432) |
|
| > loss: 0.37666434049606323 (0.3694539158294597) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3312 (0.3247146815061568) |
|
| > loader_time: 0.034 (0.03108005424340566) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:45:55 -- STEP: 290/1935 -- GLOBAL_STEP: 11900[0m |
|
| > loss_text_ce: 0.04076746851205826 (0.04313386290751652) |
|
| > loss_mel_ce: 2.9685192108154297 (2.913753234106919) |
|
| > loss: 0.3761608302593231 (0.3696108878686508) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3263 (0.32405549164476055) |
|
| > loader_time: 0.0359 (0.03136325704640355) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:46:21 -- STEP: 340/1935 -- GLOBAL_STEP: 11950[0m |
|
| > loss_text_ce: 0.042949117720127106 (0.04311119030941932) |
|
| > loss_mel_ce: 2.8615546226501465 (2.914442921386046) |
|
| > loss: 0.3630629777908325 (0.3696942648466893) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2915 (0.32419666963465077) |
|
| > loader_time: 0.0323 (0.0309599189197316) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:46:47 -- STEP: 390/1935 -- GLOBAL_STEP: 12000[0m |
|
| > loss_text_ce: 0.041277479380369186 (0.04305513256635417) |
|
| > loss_mel_ce: 2.875748634338379 (2.914253802788564) |
|
| > loss: 0.36462825536727905 (0.36966361754979826) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3204 (0.3244298562025413) |
|
| > loader_time: 0.0338 (0.031302908139351104) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:47:13 -- STEP: 440/1935 -- GLOBAL_STEP: 12050[0m |
|
| > loss_text_ce: 0.041693832725286484 (0.043020517480644276) |
|
| > loss_mel_ce: 2.9659171104431152 (2.91482508616014) |
|
| > loss: 0.3759513795375824 (0.3697307010265913) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2867 (0.323825650323521) |
|
| > loader_time: 0.0333 (0.031570641560987996) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:47:39 -- STEP: 490/1935 -- GLOBAL_STEP: 12100[0m |
|
| > loss_text_ce: 0.042212262749671936 (0.043030508235096895) |
|
| > loss_mel_ce: 2.8896102905273438 (2.9138269010855233) |
|
| > loss: 0.3664778172969818 (0.3696071767077153) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3429 (0.32357139636059196) |
|
| > loader_time: 0.0348 (0.0316686659443135) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:48:05 -- STEP: 540/1935 -- GLOBAL_STEP: 12150[0m |
|
| > loss_text_ce: 0.04176709055900574 (0.04303929204880085) |
|
| > loss_mel_ce: 2.825913667678833 (2.9118250692332235) |
|
| > loss: 0.35846009850502014 (0.36935804541464196) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2964 (0.32321027252409185) |
|
| > loader_time: 0.0324 (0.031841472784678145) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:48:31 -- STEP: 590/1935 -- GLOBAL_STEP: 12200[0m |
|
| > loss_text_ce: 0.042140115052461624 (0.043039566770953636) |
|
| > loss_mel_ce: 2.9555256366729736 (2.9124544022446983) |
|
| > loss: 0.3747082054615021 (0.3694367463305845) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3363 (0.3238657377533993) |
|
| > loader_time: 0.0341 (0.03201303158776236) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:48:57 -- STEP: 640/1935 -- GLOBAL_STEP: 12250[0m |
|
| > loss_text_ce: 0.04460030421614647 (0.043043606245191754) |
|
| > loss_mel_ce: 2.909742593765259 (2.9115107588469997) |
|
| > loss: 0.36929285526275635 (0.3693192957434806) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3262 (0.32360272221267217) |
|
| > loader_time: 0.0343 (0.032144595682621) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:49:23 -- STEP: 690/1935 -- GLOBAL_STEP: 12300[0m |
|
| > loss_text_ce: 0.0456831157207489 (0.0430246276052102) |
|
| > loss_mel_ce: 2.9896531105041504 (2.912226751921837) |
|
| > loss: 0.3794170320034027 (0.3694064226703366) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3174 (0.32364599359208274) |
|
| > loader_time: 0.0332 (0.03231216720912768) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:49:50 -- STEP: 740/1935 -- GLOBAL_STEP: 12350[0m |
|
| > loss_text_ce: 0.04194983094930649 (0.04301686749063636) |
|
| > loss_mel_ce: 2.955432653427124 (2.912259944709575) |
|
| > loss: 0.3746728003025055 (0.3694096017125489) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3377 (0.32424811350332733) |
|
| > loader_time: 0.0331 (0.03237776111912081) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:50:16 -- STEP: 790/1935 -- GLOBAL_STEP: 12400[0m |
|
| > loss_text_ce: 0.04218154400587082 (0.0430115862549106) |
|
| > loss_mel_ce: 2.8713128566741943 (2.9122450795354755) |
|
| > loss: 0.3641867935657501 (0.3694070834902267) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2929 (0.32429941636097587) |
|
| > loader_time: 0.0325 (0.03244538337369507) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:50:42 -- STEP: 840/1935 -- GLOBAL_STEP: 12450[0m |
|
| > loss_text_ce: 0.04446200653910637 (0.043010742753921534) |
|
| > loss_mel_ce: 2.955151319503784 (2.91212802472569) |
|
| > loss: 0.37495166063308716 (0.3693923460585729) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3184 (0.32448361005101894) |
|
| > loader_time: 0.0387 (0.03252395363081071) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:51:08 -- STEP: 890/1935 -- GLOBAL_STEP: 12500[0m |
|
| > loss_text_ce: 0.042331743985414505 (0.04300470673217534) |
|
| > loss_mel_ce: 2.890871047973633 (2.911878080046582) |
|
| > loss: 0.3666503429412842 (0.36936034841483883) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3426 (0.3245459403884545) |
|
| > loader_time: 0.0338 (0.03261792686548128) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:51:34 -- STEP: 940/1935 -- GLOBAL_STEP: 12550[0m |
|
| > loss_text_ce: 0.04262128844857216 (0.042996740131143574) |
|
| > loss_mel_ce: 2.8502330780029297 (2.910956032225429) |
|
| > loss: 0.36160680651664734 (0.36924409672934944) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2988 (0.3244162896846204) |
|
| > loader_time: 0.0326 (0.03271353625236676) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:52:00 -- STEP: 990/1935 -- GLOBAL_STEP: 12600[0m |
|
| > loss_text_ce: 0.04130048677325249 (0.042995597000675984) |
|
| > loss_mel_ce: 2.967358350753784 (2.9107214460469284) |
|
| > loss: 0.3760823607444763 (0.3692146306387101) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3483 (0.32413058088283353) |
|
| > loader_time: 0.0348 (0.03280247197006688) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:52:26 -- STEP: 1040/1935 -- GLOBAL_STEP: 12650[0m |
|
| > loss_text_ce: 0.041832469403743744 (0.042976733683966685) |
|
| > loss_mel_ce: 2.9459750652313232 (2.911365050077441) |
|
| > loss: 0.37347593903541565 (0.3692927231009189) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2848 (0.3241628497838975) |
|
| > loader_time: 0.0346 (0.03286925416726333) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:52:52 -- STEP: 1090/1935 -- GLOBAL_STEP: 12700[0m |
|
| > loss_text_ce: 0.0432749018073082 (0.04296350443581922) |
|
| > loss_mel_ce: 2.981794595718384 (2.9113291668235735) |
|
| > loss: 0.3781336843967438 (0.3692865840338786) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3356 (0.3243501518844464) |
|
| > loader_time: 0.0341 (0.032956230749777715) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:53:18 -- STEP: 1140/1935 -- GLOBAL_STEP: 12750[0m |
|
| > loss_text_ce: 0.04380502924323082 (0.04296307607570237) |
|
| > loss_mel_ce: 2.904792547225952 (2.911437543651517) |
|
| > loss: 0.36857470870018005 (0.36930007754187816) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2772 (0.32450635475024847) |
|
| > loader_time: 0.0355 (0.03301760376545424) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:53:44 -- STEP: 1190/1935 -- GLOBAL_STEP: 12800[0m |
|
| > loss_text_ce: 0.042392831295728683 (0.0429526595383131) |
|
| > loss_mel_ce: 2.94791841506958 (2.911380319435058) |
|
| > loss: 0.37378889322280884 (0.3692916224483681) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2717 (0.32445727897291426) |
|
| > loader_time: 0.0345 (0.0330823647875746) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:54:10 -- STEP: 1240/1935 -- GLOBAL_STEP: 12850[0m |
|
| > loss_text_ce: 0.043120644986629486 (0.04295356641974177) |
|
| > loss_mel_ce: 2.9774723052978516 (2.9101735182346866) |
|
| > loss: 0.3775741159915924 (0.3691408856261157) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3067 (0.3244074708031068) |
|
| > loader_time: 0.0343 (0.03314951869749255) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:54:37 -- STEP: 1290/1935 -- GLOBAL_STEP: 12900[0m |
|
| > loss_text_ce: 0.04369216039776802 (0.0429450589383757) |
|
| > loss_mel_ce: 2.8792307376861572 (2.9096555497295182) |
|
| > loss: 0.3653653562068939 (0.3690750761780622) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2975 (0.3244216588116432) |
|
| > loader_time: 0.0335 (0.03324584887009262) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:55:03 -- STEP: 1340/1935 -- GLOBAL_STEP: 12950[0m |
|
| > loss_text_ce: 0.04287298768758774 (0.04294696152210231) |
|
| > loss_mel_ce: 2.911001443862915 (2.909110191153057) |
|
| > loss: 0.3692342936992645 (0.3690071441344358) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3327 (0.3245281616253641) |
|
| > loader_time: 0.036 (0.033314657033379415) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:55:29 -- STEP: 1390/1935 -- GLOBAL_STEP: 13000[0m |
|
| > loss_text_ce: 0.04271633177995682 (0.042942577021585035) |
|
| > loss_mel_ce: 2.9080984592437744 (2.9092204052767316) |
|
| > loss: 0.36885184049606323 (0.3690203728650112) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3095 (0.32474364942783934) |
|
| > loader_time: 0.0333 (0.033344889544754584) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:55:56 -- STEP: 1440/1935 -- GLOBAL_STEP: 13050[0m |
|
| > loss_text_ce: 0.043519239872694016 (0.04292744380266712) |
|
| > loss_mel_ce: 2.8434293270111084 (2.908897624744311) |
|
| > loss: 0.36086857318878174 (0.368978133652773) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2932 (0.32479051417774607) |
|
| > loader_time: 0.0318 (0.03337802009450065) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:56:22 -- STEP: 1490/1935 -- GLOBAL_STEP: 13100[0m |
|
| > loss_text_ce: 0.03977774828672409 (0.04291369089444212) |
|
| > loss_mel_ce: 2.896620750427246 (2.9088833690489717) |
|
| > loss: 0.36704981327056885 (0.3689746326248116) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3333 (0.3248570650215915) |
|
| > loader_time: 0.0348 (0.0333909966001575) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:56:47 -- STEP: 1540/1935 -- GLOBAL_STEP: 13150[0m |
|
| > loss_text_ce: 0.04455053061246872 (0.04290079162879424) |
|
| > loss_mel_ce: 2.8935580253601074 (2.9090328136047776) |
|
| > loss: 0.3672635555267334 (0.3689917008598128) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3183 (0.3246778895328569) |
|
| > loader_time: 0.0345 (0.03337297176385861) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:57:13 -- STEP: 1590/1935 -- GLOBAL_STEP: 13200[0m |
|
| > loss_text_ce: 0.04482114315032959 (0.04290175912843185) |
|
| > loss_mel_ce: 2.9599056243896484 (2.9091753149932305) |
|
| > loss: 0.37559086084365845 (0.3690096344962808) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3178 (0.3246413287876537) |
|
| > loader_time: 0.0341 (0.03338980689738534) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:57:40 -- STEP: 1640/1935 -- GLOBAL_STEP: 13250[0m |
|
| > loss_text_ce: 0.044805485755205154 (0.04289589937004011) |
|
| > loss_mel_ce: 2.8616838455200195 (2.908611887838784) |
|
| > loss: 0.36331117153167725 (0.36893847364841426) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2552 (0.32468442931407837) |
|
| > loader_time: 0.0329 (0.03343817707968925) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:58:06 -- STEP: 1690/1935 -- GLOBAL_STEP: 13300[0m |
|
| > loss_text_ce: 0.04361748695373535 (0.04289629328444865) |
|
| > loss_mel_ce: 2.93487811088562 (2.908442568214689) |
|
| > loss: 0.37231194972991943 (0.3689173579392347) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2762 (0.32461826674331573) |
|
| > loader_time: 0.0335 (0.033470340452250696) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:58:32 -- STEP: 1740/1935 -- GLOBAL_STEP: 13350[0m |
|
| > loss_text_ce: 0.043585292994976044 (0.04289177032722824) |
|
| > loss_mel_ce: 2.882695436477661 (2.9083563523731026) |
|
| > loss: 0.3657850921154022 (0.36890601556876595) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3859 (0.32464348132582915) |
|
| > loader_time: 0.0371 (0.033492709987465036) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:58:58 -- STEP: 1790/1935 -- GLOBAL_STEP: 13400[0m |
|
| > loss_text_ce: 0.04401775076985359 (0.04289458492281717) |
|
| > loss_mel_ce: 2.9097912311553955 (2.90848724682238) |
|
| > loss: 0.3692261278629303 (0.3689227292158083) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.339 (0.32469984765825327) |
|
| > loader_time: 0.035 (0.033522909846385826) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:59:24 -- STEP: 1840/1935 -- GLOBAL_STEP: 13450[0m |
|
| > loss_text_ce: 0.040951356291770935 (0.04289258976224) |
|
| > loss_mel_ce: 2.7110865116119385 (2.908437120137009) |
|
| > loss: 0.34400472044944763 (0.36891621399832797) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2893 (0.3247542512157689) |
|
| > loader_time: 0.0338 (0.033551661864571) |
|
|
|
|
|
[1m --> TIME: 2024-12-23 18:59:50 -- STEP: 1890/1935 -- GLOBAL_STEP: 13500[0m |
|
| > loss_text_ce: 0.04326050356030464 (0.0428974959112349) |
|
| > loss_mel_ce: 2.8731529712677 (2.9082712471169785) |
|
| > loss: 0.36455169320106506 (0.3688960931130817) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3378 (0.3246376347920251) |
|
| > loader_time: 0.0352 (0.03357641696929936) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[91m 0.11718952174138544 [0m(+0.00033069379401944043) |
|
| > avg_loss_text_ce:[92m 0.041147362083347155 [0m(-0.00023140923844446049) |
|
| > avg_loss_mel_ce:[92m 2.973366911965186 [0m(-0.004587167441244766) |
|
| > avg_loss:[92m 3.0145142740673507 [0m(-0.004818575550809001) |
|
|
|
> BEST MODEL : /checkpoints/GPT_XTTS_FT-December-23-2024_04+57PM-0000000/best_model_13545.pth |
|
! Run is kept in /checkpoints/GPT_XTTS_FT-December-23-2024_04+57PM-0000000 |
|
|