|
|
|
|
|
> Training Environment:
|
|
|
|
|
|
| > Current device: 0
|
|
|
|
|
|
| > Num. of GPUs: 1
|
|
|
|
|
|
| > Num. of CPUs: 2
|
|
|
|
|
|
| > Num. of Torch Threads: 1
|
|
|
|
|
|
| > Torch seed: 54321
|
|
|
|
|
|
| > Torch CUDNN: True
|
|
|
|
|
|
| > Torch CUDNN deterministic: False
|
|
|
|
|
|
| > Torch CUDNN benchmark: False
|
|
|
|
|
|
> Start Tensorboard: tensorboard --logdir=output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
> Model has 28259417 parameters
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 0/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:01:10) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0/2 -- GLOBAL_STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 28.89919 (28.89919)
|
|
|
|
|
|
| > postnet_loss: 31.12150 (31.12150)
|
|
|
|
|
|
| > stopnet_loss: 0.77770 (0.77770)
|
|
|
|
|
|
| > loss: 15.78287 (15.78287)
|
|
|
|
|
|
| > align_error: 0.95760 (0.95760)
|
|
|
|
|
|
| > grad_norm: 7.37357 (7.37357)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 2.27920 (2.27921)
|
|
|
|
|
|
| > loader_time: 1.91060 (1.91060)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.50797 (25.50797)
|
|
|
|
|
|
| > postnet_loss: 25.51569 (25.51569)
|
|
|
|
|
|
| > stopnet_loss: 0.79791 (0.79791)
|
|
|
|
|
|
| > loss: 13.55383 (13.55383)
|
|
|
|
|
|
| > align_error: 0.93281 (0.93281)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time: 0.39936 [0m(+0.00000)
|
|
|
|
|
|
| > avg_decoder_loss: 25.50797 [0m(+0.00000)
|
|
|
|
|
|
| > avg_postnet_loss: 25.51569 [0m(+0.00000)
|
|
|
|
|
|
| > avg_stopnet_loss: 0.79791 [0m(+0.00000)
|
|
|
|
|
|
| > avg_loss: 13.55383 [0m(+0.00000)
|
|
|
|
|
|
| > avg_align_error: 0.93281 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_2.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 1/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:01:26) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.50425 (25.50425)
|
|
|
|
|
|
| > postnet_loss: 25.51034 (25.51034)
|
|
|
|
|
|
| > stopnet_loss: 0.79768 (0.79768)
|
|
|
|
|
|
| > loss: 13.55133 (13.55133)
|
|
|
|
|
|
| > align_error: 0.93281 (0.93281)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.48173 [0m(+0.08238)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.50425 [0m(-0.00372)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.51034 [0m(-0.00536)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79768 [0m(-0.00024)
|
|
|
|
|
|
| > avg_loss:[92m 13.55133 [0m(-0.00250)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93281 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_4.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 2/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:01:43) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.50184 (25.50184)
|
|
|
|
|
|
| > postnet_loss: 25.50626 (25.50626)
|
|
|
|
|
|
| > stopnet_loss: 0.79740 (0.79740)
|
|
|
|
|
|
| > loss: 13.54942 (13.54942)
|
|
|
|
|
|
| > align_error: 0.93281 (0.93281)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.37810 [0m(-0.10363)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.50184 [0m(-0.00241)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.50626 [0m(-0.00407)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79740 [0m(-0.00028)
|
|
|
|
|
|
| > avg_loss:[92m 13.54942 [0m(-0.00191)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93281 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_6.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 3/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:01:59) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.50009 (25.50009)
|
|
|
|
|
|
| > postnet_loss: 25.50450 (25.50450)
|
|
|
|
|
|
| > stopnet_loss: 0.79708 (0.79708)
|
|
|
|
|
|
| > loss: 13.54823 (13.54823)
|
|
|
|
|
|
| > align_error: 0.93280 (0.93280)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.40665 [0m(+0.02855)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.50009 [0m(-0.00175)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.50450 [0m(-0.00176)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79708 [0m(-0.00031)
|
|
|
|
|
|
| > avg_loss:[92m 13.54823 [0m(-0.00119)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93280 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_8.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 4/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:02:16) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49852 (25.49852)
|
|
|
|
|
|
| > postnet_loss: 25.50244 (25.50244)
|
|
|
|
|
|
| > stopnet_loss: 0.79684 (0.79684)
|
|
|
|
|
|
| > loss: 13.54708 (13.54708)
|
|
|
|
|
|
| > align_error: 0.93280 (0.93280)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.34384 [0m(-0.06281)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49852 [0m(-0.00157)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.50244 [0m(-0.00206)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79684 [0m(-0.00024)
|
|
|
|
|
|
| > avg_loss:[92m 13.54708 [0m(-0.00115)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93280 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_10.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 5/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:02:31) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49740 (25.49740)
|
|
|
|
|
|
| > postnet_loss: 25.50116 (25.50116)
|
|
|
|
|
|
| > stopnet_loss: 0.79664 (0.79664)
|
|
|
|
|
|
| > loss: 13.54628 (13.54628)
|
|
|
|
|
|
| > align_error: 0.93280 (0.93280)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.35995 [0m(+0.01611)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49740 [0m(-0.00112)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.50116 [0m(-0.00128)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79664 [0m(-0.00020)
|
|
|
|
|
|
| > avg_loss:[92m 13.54628 [0m(-0.00080)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93280 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_12.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 6/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:02:46) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49664 (25.49664)
|
|
|
|
|
|
| > postnet_loss: 25.50632 (25.50632)
|
|
|
|
|
|
| > stopnet_loss: 0.79646 (0.79646)
|
|
|
|
|
|
| > loss: 13.54720 (13.54720)
|
|
|
|
|
|
| > align_error: 0.93279 (0.93279)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.34161 [0m(-0.01834)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49664 [0m(-0.00075)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.50632 [0m(+0.00516)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79646 [0m(-0.00019)
|
|
|
|
|
|
| > avg_loss:[91m 13.54720 [0m(+0.00091)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93279 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 7/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:02:58) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49627 (25.49627)
|
|
|
|
|
|
| > postnet_loss: 25.51181 (25.51181)
|
|
|
|
|
|
| > stopnet_loss: 0.79645 (0.79645)
|
|
|
|
|
|
| > loss: 13.54846 (13.54846)
|
|
|
|
|
|
| > align_error: 0.93279 (0.93279)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.51325 [0m(+0.17164)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49627 [0m(-0.00038)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.51181 [0m(+0.00550)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79645 [0m(-0.00001)
|
|
|
|
|
|
| > avg_loss:[91m 13.54846 [0m(+0.00127)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93279 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 8/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:03:09) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49568 (25.49568)
|
|
|
|
|
|
| > postnet_loss: 25.51995 (25.51995)
|
|
|
|
|
|
| > stopnet_loss: 0.79641 (0.79641)
|
|
|
|
|
|
| > loss: 13.55032 (13.55032)
|
|
|
|
|
|
| > align_error: 0.93279 (0.93279)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.47709 [0m(-0.03616)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49568 [0m(-0.00059)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.51995 [0m(+0.00814)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79641 [0m(-0.00004)
|
|
|
|
|
|
| > avg_loss:[91m 13.55032 [0m(+0.00185)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93279 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 9/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:03:23) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49526 (25.49526)
|
|
|
|
|
|
| > postnet_loss: 25.53222 (25.53222)
|
|
|
|
|
|
| > stopnet_loss: 0.79644 (0.79644)
|
|
|
|
|
|
| > loss: 13.55331 (13.55331)
|
|
|
|
|
|
| > align_error: 0.93278 (0.93278)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.37861 [0m(-0.09847)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49526 [0m(-0.00042)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.53222 [0m(+0.01227)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79644 [0m(+0.00003)
|
|
|
|
|
|
| > avg_loss:[91m 13.55331 [0m(+0.00299)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93278 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 10/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:03:35) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49488 (25.49488)
|
|
|
|
|
|
| > postnet_loss: 25.54352 (25.54352)
|
|
|
|
|
|
| > stopnet_loss: 0.79661 (0.79661)
|
|
|
|
|
|
| > loss: 13.55621 (13.55621)
|
|
|
|
|
|
| > align_error: 0.93278 (0.93278)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.40135 [0m(+0.02274)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49488 [0m(-0.00038)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.54352 [0m(+0.01130)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79661 [0m(+0.00017)
|
|
|
|
|
|
| > avg_loss:[91m 13.55621 [0m(+0.00290)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93278 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 11/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:03:47) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49456 (25.49456)
|
|
|
|
|
|
| > postnet_loss: 25.56388 (25.56388)
|
|
|
|
|
|
| > stopnet_loss: 0.79683 (0.79683)
|
|
|
|
|
|
| > loss: 13.56144 (13.56144)
|
|
|
|
|
|
| > align_error: 0.93278 (0.93278)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.36292 [0m(-0.03843)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49456 [0m(-0.00031)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.56388 [0m(+0.02036)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79683 [0m(+0.00022)
|
|
|
|
|
|
| > avg_loss:[91m 13.56144 [0m(+0.00524)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93278 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 12/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:03:59) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 1/2 -- GLOBAL_STEP: 25[0m
|
|
|
|
|
|
| > decoder_loss: 30.87367 (30.87367)
|
|
|
|
|
|
| > postnet_loss: 33.26153 (33.26153)
|
|
|
|
|
|
| > stopnet_loss: 0.78800 (0.78800)
|
|
|
|
|
|
| > loss: 16.82180 (16.82180)
|
|
|
|
|
|
| > align_error: 0.96976 (0.96976)
|
|
|
|
|
|
| > grad_norm: 9.20706 (9.20706)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 2.63590 (2.63595)
|
|
|
|
|
|
| > loader_time: 0.00610 (0.00611)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49454 (25.49454)
|
|
|
|
|
|
| > postnet_loss: 25.58455 (25.58455)
|
|
|
|
|
|
| > stopnet_loss: 0.79701 (0.79701)
|
|
|
|
|
|
| > loss: 13.56678 (13.56678)
|
|
|
|
|
|
| > align_error: 0.93278 (0.93278)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.35723 [0m(-0.00569)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49454 [0m(-0.00002)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.58455 [0m(+0.02067)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79701 [0m(+0.00018)
|
|
|
|
|
|
| > avg_loss:[91m 13.56678 [0m(+0.00534)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93278 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 13/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:04:11) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49424 (25.49424)
|
|
|
|
|
|
| > postnet_loss: 25.60576 (25.60576)
|
|
|
|
|
|
| > stopnet_loss: 0.79718 (0.79718)
|
|
|
|
|
|
| > loss: 13.57218 (13.57218)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.01607 [0m(+0.65885)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49424 [0m(-0.00030)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.60576 [0m(+0.02122)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79718 [0m(+0.00017)
|
|
|
|
|
|
| > avg_loss:[91m 13.57218 [0m(+0.00540)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93277 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 14/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:04:28) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49384 (25.49384)
|
|
|
|
|
|
| > postnet_loss: 25.62736 (25.62736)
|
|
|
|
|
|
| > stopnet_loss: 0.79730 (0.79730)
|
|
|
|
|
|
| > loss: 13.57760 (13.57760)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.49513 [0m(-0.52094)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49384 [0m(-0.00040)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.62736 [0m(+0.02160)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79730 [0m(+0.00012)
|
|
|
|
|
|
| > avg_loss:[91m 13.57760 [0m(+0.00542)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93277 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 15/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:04:41) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49370 (25.49370)
|
|
|
|
|
|
| > postnet_loss: 25.64535 (25.64535)
|
|
|
|
|
|
| > stopnet_loss: 0.79763 (0.79763)
|
|
|
|
|
|
| > loss: 13.58240 (13.58240)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.36358 [0m(-0.13155)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49370 [0m(-0.00014)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.64535 [0m(+0.01799)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79763 [0m(+0.00033)
|
|
|
|
|
|
| > avg_loss:[91m 13.58240 [0m(+0.00479)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93277 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 16/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:04:53) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49369 (25.49369)
|
|
|
|
|
|
| > postnet_loss: 25.66530 (25.66530)
|
|
|
|
|
|
| > stopnet_loss: 0.79777 (0.79777)
|
|
|
|
|
|
| > loss: 13.58752 (13.58752)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.36918 [0m(+0.00559)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49369 [0m(-0.00002)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.66530 [0m(+0.01995)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79777 [0m(+0.00014)
|
|
|
|
|
|
| > avg_loss:[91m 13.58752 [0m(+0.00512)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93277 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 17/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:05:05) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49309 (25.49309)
|
|
|
|
|
|
| > postnet_loss: 25.68219 (25.68219)
|
|
|
|
|
|
| > stopnet_loss: 0.79789 (0.79789)
|
|
|
|
|
|
| > loss: 13.59171 (13.59171)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.35864 [0m(-0.01053)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49309 [0m(-0.00060)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.68219 [0m(+0.01690)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79789 [0m(+0.00012)
|
|
|
|
|
|
| > avg_loss:[91m 13.59171 [0m(+0.00419)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93277 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 18/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:05:17) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49240 (25.49240)
|
|
|
|
|
|
| > postnet_loss: 25.70145 (25.70145)
|
|
|
|
|
|
| > stopnet_loss: 0.79788 (0.79788)
|
|
|
|
|
|
| > loss: 13.59634 (13.59634)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.35795 [0m(-0.00069)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49240 [0m(-0.00068)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.70145 [0m(+0.01925)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79788 [0m(-0.00001)
|
|
|
|
|
|
| > avg_loss:[91m 13.59634 [0m(+0.00463)
|
|
|
|
|
|
| > avg_align_error:[92m 0.93277 [0m(-0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 19/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:05:30) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49167 (25.49167)
|
|
|
|
|
|
| > postnet_loss: 25.71669 (25.71669)
|
|
|
|
|
|
| > stopnet_loss: 0.79810 (0.79810)
|
|
|
|
|
|
| > loss: 13.60019 (13.60019)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.49330 [0m(+0.13535)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49167 [0m(-0.00073)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.71669 [0m(+0.01524)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79810 [0m(+0.00022)
|
|
|
|
|
|
| > avg_loss:[91m 13.60019 [0m(+0.00385)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93277 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 20/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:05:42) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49085 (25.49085)
|
|
|
|
|
|
| > postnet_loss: 25.72650 (25.72650)
|
|
|
|
|
|
| > stopnet_loss: 0.79813 (0.79813)
|
|
|
|
|
|
| > loss: 13.60247 (13.60247)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.35988 [0m(-0.13342)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49085 [0m(-0.00082)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.72650 [0m(+0.00981)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79813 [0m(+0.00004)
|
|
|
|
|
|
| > avg_loss:[91m 13.60247 [0m(+0.00228)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93277 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 21/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:05:55) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.49057 (25.49057)
|
|
|
|
|
|
| > postnet_loss: 25.73598 (25.73598)
|
|
|
|
|
|
| > stopnet_loss: 0.79826 (0.79826)
|
|
|
|
|
|
| > loss: 13.60490 (13.60490)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.38151 [0m(+0.02163)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.49057 [0m(-0.00028)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.73598 [0m(+0.00948)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79826 [0m(+0.00013)
|
|
|
|
|
|
| > avg_loss:[91m 13.60490 [0m(+0.00243)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93277 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 22/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:06:07) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.48928 (25.48928)
|
|
|
|
|
|
| > postnet_loss: 25.73852 (25.73852)
|
|
|
|
|
|
| > stopnet_loss: 0.79828 (0.79828)
|
|
|
|
|
|
| > loss: 13.60523 (13.60523)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.36862 [0m(-0.01289)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.48928 [0m(-0.00129)
|
|
|
|
|
|
| > avg_postnet_loss:[91m 25.73852 [0m(+0.00254)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79828 [0m(+0.00002)
|
|
|
|
|
|
| > avg_loss:[91m 13.60523 [0m(+0.00033)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93277 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 23/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:06:18) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.48816 (25.48816)
|
|
|
|
|
|
| > postnet_loss: 25.73845 (25.73845)
|
|
|
|
|
|
| > stopnet_loss: 0.79823 (0.79823)
|
|
|
|
|
|
| > loss: 13.60488 (13.60488)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.74967 [0m(+0.38104)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.48816 [0m(-0.00112)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.73845 [0m(-0.00007)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79823 [0m(-0.00005)
|
|
|
|
|
|
| > avg_loss:[92m 13.60488 [0m(-0.00035)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93277 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 24/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:06:32) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.48675 (25.48675)
|
|
|
|
|
|
| > postnet_loss: 25.73373 (25.73373)
|
|
|
|
|
|
| > stopnet_loss: 0.79814 (0.79814)
|
|
|
|
|
|
| > loss: 13.60326 (13.60326)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.35577 [0m(-0.39389)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.48675 [0m(-0.00141)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.73373 [0m(-0.00472)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79814 [0m(-0.00009)
|
|
|
|
|
|
| > avg_loss:[92m 13.60326 [0m(-0.00162)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93277 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 25/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:06:45) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0/2 -- GLOBAL_STEP: 50[0m
|
|
|
|
|
|
| > decoder_loss: 28.89064 (28.89064)
|
|
|
|
|
|
| > postnet_loss: 31.06001 (31.06001)
|
|
|
|
|
|
| > stopnet_loss: 0.77744 (0.77744)
|
|
|
|
|
|
| > loss: 15.76510 (15.76510)
|
|
|
|
|
|
| > align_error: 0.95761 (0.95761)
|
|
|
|
|
|
| > grad_norm: 7.38501 (7.38501)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 0.82440 (0.82440)
|
|
|
|
|
|
| > loader_time: 1.38490 (1.38492)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.48573 (25.48573)
|
|
|
|
|
|
| > postnet_loss: 25.72834 (25.72834)
|
|
|
|
|
|
| > stopnet_loss: 0.79824 (0.79824)
|
|
|
|
|
|
| > loss: 13.60176 (13.60176)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.53864 [0m(+0.18287)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.48573 [0m(-0.00102)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.72834 [0m(-0.00538)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79824 [0m(+0.00010)
|
|
|
|
|
|
| > avg_loss:[92m 13.60176 [0m(-0.00150)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93277 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 26/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:06:57) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.48450 (25.48450)
|
|
|
|
|
|
| > postnet_loss: 25.71525 (25.71525)
|
|
|
|
|
|
| > stopnet_loss: 0.79823 (0.79823)
|
|
|
|
|
|
| > loss: 13.59817 (13.59817)
|
|
|
|
|
|
| > align_error: 0.93277 (0.93277)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.35837 [0m(-0.18027)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.48450 [0m(-0.00123)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.71525 [0m(-0.01309)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79823 [0m(-0.00001)
|
|
|
|
|
|
| > avg_loss:[92m 13.59817 [0m(-0.00359)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93277 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 27/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:07:10) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.48305 (25.48305)
|
|
|
|
|
|
| > postnet_loss: 25.69920 (25.69920)
|
|
|
|
|
|
| > stopnet_loss: 0.79810 (0.79810)
|
|
|
|
|
|
| > loss: 13.59366 (13.59366)
|
|
|
|
|
|
| > align_error: 0.93278 (0.93278)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.39568 [0m(+0.03732)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.48305 [0m(-0.00145)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.69920 [0m(-0.01606)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79810 [0m(-0.00013)
|
|
|
|
|
|
| > avg_loss:[92m 13.59366 [0m(-0.00451)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93278 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 28/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:07:22) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.48158 (25.48158)
|
|
|
|
|
|
| > postnet_loss: 25.68590 (25.68590)
|
|
|
|
|
|
| > stopnet_loss: 0.79803 (0.79803)
|
|
|
|
|
|
| > loss: 13.58990 (13.58990)
|
|
|
|
|
|
| > align_error: 0.93278 (0.93278)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.77588 [0m(+0.38020)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.48158 [0m(-0.00147)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.68590 [0m(-0.01330)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79803 [0m(-0.00007)
|
|
|
|
|
|
| > avg_loss:[92m 13.58990 [0m(-0.00376)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93278 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 29/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:07:34) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.48022 (25.48022)
|
|
|
|
|
|
| > postnet_loss: 25.66250 (25.66250)
|
|
|
|
|
|
| > stopnet_loss: 0.79797 (0.79797)
|
|
|
|
|
|
| > loss: 13.58365 (13.58365)
|
|
|
|
|
|
| > align_error: 0.93278 (0.93278)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.36242 [0m(-0.41346)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.48022 [0m(-0.00136)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.66250 [0m(-0.02340)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79797 [0m(-0.00006)
|
|
|
|
|
|
| > avg_loss:[92m 13.58365 [0m(-0.00624)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93278 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 30/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:07:46) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.47774 (25.47774)
|
|
|
|
|
|
| > postnet_loss: 25.63940 (25.63940)
|
|
|
|
|
|
| > stopnet_loss: 0.79772 (0.79772)
|
|
|
|
|
|
| > loss: 13.57700 (13.57700)
|
|
|
|
|
|
| > align_error: 0.93278 (0.93278)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.49328 [0m(+0.13086)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.47774 [0m(-0.00248)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.63940 [0m(-0.02310)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79772 [0m(-0.00025)
|
|
|
|
|
|
| > avg_loss:[92m 13.57700 [0m(-0.00665)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93278 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 31/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:07:58) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.47592 (25.47592)
|
|
|
|
|
|
| > postnet_loss: 25.61328 (25.61328)
|
|
|
|
|
|
| > stopnet_loss: 0.79757 (0.79757)
|
|
|
|
|
|
| > loss: 13.56987 (13.56987)
|
|
|
|
|
|
| > align_error: 0.93278 (0.93278)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.36018 [0m(-0.13311)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.47592 [0m(-0.00182)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.61328 [0m(-0.02612)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79757 [0m(-0.00015)
|
|
|
|
|
|
| > avg_loss:[92m 13.56987 [0m(-0.00714)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93278 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 32/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:08:11) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.47375 (25.47375)
|
|
|
|
|
|
| > postnet_loss: 25.57905 (25.57905)
|
|
|
|
|
|
| > stopnet_loss: 0.79735 (0.79735)
|
|
|
|
|
|
| > loss: 13.56055 (13.56055)
|
|
|
|
|
|
| > align_error: 0.93278 (0.93278)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.37231 [0m(+0.01213)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.47375 [0m(-0.00217)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.57905 [0m(-0.03423)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79735 [0m(-0.00022)
|
|
|
|
|
|
| > avg_loss:[92m 13.56055 [0m(-0.00932)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93278 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 33/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:08:24) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.47235 (25.47235)
|
|
|
|
|
|
| > postnet_loss: 25.54000 (25.54000)
|
|
|
|
|
|
| > stopnet_loss: 0.79739 (0.79739)
|
|
|
|
|
|
| > loss: 13.55048 (13.55048)
|
|
|
|
|
|
| > align_error: 0.93279 (0.93279)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.37305 [0m(+0.00075)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.47235 [0m(-0.00139)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.54000 [0m(-0.03905)
|
|
|
|
|
|
| > avg_stopnet_loss:[91m 0.79739 [0m(+0.00004)
|
|
|
|
|
|
| > avg_loss:[92m 13.55048 [0m(-0.01007)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93279 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 34/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:08:35) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.47011 (25.47011)
|
|
|
|
|
|
| > postnet_loss: 25.49636 (25.49636)
|
|
|
|
|
|
| > stopnet_loss: 0.79721 (0.79721)
|
|
|
|
|
|
| > loss: 13.53883 (13.53883)
|
|
|
|
|
|
| > align_error: 0.93279 (0.93279)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.42491 [0m(+0.05186)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.47011 [0m(-0.00224)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.49636 [0m(-0.04364)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79721 [0m(-0.00018)
|
|
|
|
|
|
| > avg_loss:[92m 13.53883 [0m(-0.01165)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93279 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_70.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 35/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:08:52) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.46753 (25.46753)
|
|
|
|
|
|
| > postnet_loss: 25.45466 (25.45466)
|
|
|
|
|
|
| > stopnet_loss: 0.79706 (0.79706)
|
|
|
|
|
|
| > loss: 13.52761 (13.52761)
|
|
|
|
|
|
| > align_error: 0.93279 (0.93279)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.36361 [0m(-0.06130)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.46753 [0m(-0.00258)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.45466 [0m(-0.04171)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79706 [0m(-0.00015)
|
|
|
|
|
|
| > avg_loss:[92m 13.52761 [0m(-0.01122)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93279 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_72.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 36/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:09:06) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.46554 (25.46554)
|
|
|
|
|
|
| > postnet_loss: 25.41213 (25.41213)
|
|
|
|
|
|
| > stopnet_loss: 0.79675 (0.79675)
|
|
|
|
|
|
| > loss: 13.51617 (13.51617)
|
|
|
|
|
|
| > align_error: 0.93279 (0.93279)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.37201 [0m(+0.00840)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.46554 [0m(-0.00200)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.41213 [0m(-0.04252)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79675 [0m(-0.00031)
|
|
|
|
|
|
| > avg_loss:[92m 13.51617 [0m(-0.01144)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93279 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_74.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 37/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:09:22) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 1/2 -- GLOBAL_STEP: 75[0m
|
|
|
|
|
|
| > decoder_loss: 30.83725 (30.83725)
|
|
|
|
|
|
| > postnet_loss: 32.78592 (32.78592)
|
|
|
|
|
|
| > stopnet_loss: 0.78673 (0.78673)
|
|
|
|
|
|
| > loss: 16.69252 (16.69252)
|
|
|
|
|
|
| > align_error: 0.96977 (0.96977)
|
|
|
|
|
|
| > grad_norm: 9.16116 (9.16116)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 2.46690 (2.46694)
|
|
|
|
|
|
| > loader_time: 0.00600 (0.00598)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.46357 (25.46357)
|
|
|
|
|
|
| > postnet_loss: 25.37069 (25.37069)
|
|
|
|
|
|
| > stopnet_loss: 0.79661 (0.79661)
|
|
|
|
|
|
| > loss: 13.50517 (13.50517)
|
|
|
|
|
|
| > align_error: 0.93280 (0.93280)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.36837 [0m(-0.00364)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.46357 [0m(-0.00197)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.37069 [0m(-0.04144)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79661 [0m(-0.00015)
|
|
|
|
|
|
| > avg_loss:[92m 13.50517 [0m(-0.01100)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93280 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_76.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 38/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:09:36) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.46148 (25.46148)
|
|
|
|
|
|
| > postnet_loss: 25.32685 (25.32685)
|
|
|
|
|
|
| > stopnet_loss: 0.79652 (0.79652)
|
|
|
|
|
|
| > loss: 13.49361 (13.49361)
|
|
|
|
|
|
| > align_error: 0.93280 (0.93280)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.36937 [0m(+0.00100)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.46148 [0m(-0.00209)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.32685 [0m(-0.04384)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79652 [0m(-0.00008)
|
|
|
|
|
|
| > avg_loss:[92m 13.49361 [0m(-0.01156)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93280 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_78.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 39/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:09:52) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.45883 (25.45883)
|
|
|
|
|
|
| > postnet_loss: 25.28308 (25.28308)
|
|
|
|
|
|
| > stopnet_loss: 0.79617 (0.79617)
|
|
|
|
|
|
| > loss: 13.48165 (13.48165)
|
|
|
|
|
|
| > align_error: 0.93280 (0.93280)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.36600 [0m(-0.00336)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.45883 [0m(-0.00265)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.28308 [0m(-0.04377)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79617 [0m(-0.00035)
|
|
|
|
|
|
| > avg_loss:[92m 13.48165 [0m(-0.01195)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93280 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_80.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 40/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:10:07) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.45604 (25.45604)
|
|
|
|
|
|
| > postnet_loss: 25.22581 (25.22581)
|
|
|
|
|
|
| > stopnet_loss: 0.79606 (0.79606)
|
|
|
|
|
|
| > loss: 13.46652 (13.46652)
|
|
|
|
|
|
| > align_error: 0.93281 (0.93281)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.37086 [0m(+0.00486)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.45604 [0m(-0.00279)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.22581 [0m(-0.05727)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79606 [0m(-0.00011)
|
|
|
|
|
|
| > avg_loss:[92m 13.46652 [0m(-0.01513)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93281 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_82.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 41/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:10:24) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.45332 (25.45332)
|
|
|
|
|
|
| > postnet_loss: 25.16945 (25.16945)
|
|
|
|
|
|
| > stopnet_loss: 0.79591 (0.79591)
|
|
|
|
|
|
| > loss: 13.45161 (13.45161)
|
|
|
|
|
|
| > align_error: 0.93281 (0.93281)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.38514 [0m(+0.01427)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.45332 [0m(-0.00272)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.16945 [0m(-0.05635)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79591 [0m(-0.00015)
|
|
|
|
|
|
| > avg_loss:[92m 13.45161 [0m(-0.01492)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93281 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_84.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 42/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:10:39) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.45007 (25.45007)
|
|
|
|
|
|
| > postnet_loss: 25.11189 (25.11189)
|
|
|
|
|
|
| > stopnet_loss: 0.79581 (0.79581)
|
|
|
|
|
|
| > loss: 13.43630 (13.43630)
|
|
|
|
|
|
| > align_error: 0.93282 (0.93282)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.36652 [0m(-0.01861)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.45007 [0m(-0.00325)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.11189 [0m(-0.05757)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79581 [0m(-0.00010)
|
|
|
|
|
|
| > avg_loss:[92m 13.43630 [0m(-0.01531)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93282 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_86.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 43/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:11:00) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.44680 (25.44680)
|
|
|
|
|
|
| > postnet_loss: 25.04398 (25.04398)
|
|
|
|
|
|
| > stopnet_loss: 0.79564 (0.79564)
|
|
|
|
|
|
| > loss: 13.41834 (13.41834)
|
|
|
|
|
|
| > align_error: 0.93282 (0.93282)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.82346 [0m(+0.45694)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.44680 [0m(-0.00327)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 25.04398 [0m(-0.06791)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79564 [0m(-0.00017)
|
|
|
|
|
|
| > avg_loss:[92m 13.41834 [0m(-0.01796)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93282 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_88.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 44/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:11:25) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.44329 (25.44329)
|
|
|
|
|
|
| > postnet_loss: 24.97836 (24.97836)
|
|
|
|
|
|
| > stopnet_loss: 0.79546 (0.79546)
|
|
|
|
|
|
| > loss: 13.40088 (13.40088)
|
|
|
|
|
|
| > align_error: 0.93283 (0.93283)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.50169 [0m(-0.32177)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.44329 [0m(-0.00351)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.97836 [0m(-0.06562)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79546 [0m(-0.00018)
|
|
|
|
|
|
| > avg_loss:[92m 13.40088 [0m(-0.01746)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93283 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_90.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 45/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:11:39) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.44047 (25.44047)
|
|
|
|
|
|
| > postnet_loss: 24.91508 (24.91508)
|
|
|
|
|
|
| > stopnet_loss: 0.79543 (0.79543)
|
|
|
|
|
|
| > loss: 13.38432 (13.38432)
|
|
|
|
|
|
| > align_error: 0.93283 (0.93283)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.49220 [0m(-0.00949)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.44047 [0m(-0.00282)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.91508 [0m(-0.06327)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79543 [0m(-0.00003)
|
|
|
|
|
|
| > avg_loss:[92m 13.38432 [0m(-0.01655)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93283 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_92.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 46/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:11:55) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.43723 (25.43723)
|
|
|
|
|
|
| > postnet_loss: 24.84629 (24.84629)
|
|
|
|
|
|
| > stopnet_loss: 0.79512 (0.79512)
|
|
|
|
|
|
| > loss: 13.36600 (13.36600)
|
|
|
|
|
|
| > align_error: 0.93283 (0.93283)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.48837 [0m(-0.00383)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.43723 [0m(-0.00324)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.84629 [0m(-0.06880)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79512 [0m(-0.00031)
|
|
|
|
|
|
| > avg_loss:[92m 13.36600 [0m(-0.01832)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93283 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_94.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 47/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:12:10) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.43436 (25.43436)
|
|
|
|
|
|
| > postnet_loss: 24.77338 (24.77338)
|
|
|
|
|
|
| > stopnet_loss: 0.79493 (0.79493)
|
|
|
|
|
|
| > loss: 13.34687 (13.34687)
|
|
|
|
|
|
| > align_error: 0.93284 (0.93284)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.48999 [0m(+0.00162)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.43436 [0m(-0.00287)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.77338 [0m(-0.07291)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79493 [0m(-0.00019)
|
|
|
|
|
|
| > avg_loss:[92m 13.34687 [0m(-0.01913)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93284 [0m(+0.00000)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_96.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 48/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:12:25) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.43090 (25.43090)
|
|
|
|
|
|
| > postnet_loss: 24.69644 (24.69644)
|
|
|
|
|
|
| > stopnet_loss: 0.79469 (0.79469)
|
|
|
|
|
|
| > loss: 13.32653 (13.32653)
|
|
|
|
|
|
| > align_error: 0.93284 (0.93284)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.49462 [0m(+0.00463)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.43090 [0m(-0.00346)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.69644 [0m(-0.07694)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79469 [0m(-0.00024)
|
|
|
|
|
|
| > avg_loss:[92m 13.32653 [0m(-0.02034)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93284 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_98.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 49/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:12:40) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.42739 (25.42739)
|
|
|
|
|
|
| > postnet_loss: 24.61805 (24.61805)
|
|
|
|
|
|
| > stopnet_loss: 0.79435 (0.79435)
|
|
|
|
|
|
| > loss: 13.30571 (13.30571)
|
|
|
|
|
|
| > align_error: 0.93285 (0.93285)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.54688 [0m(+0.05225)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.42739 [0m(-0.00351)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.61805 [0m(-0.07839)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79435 [0m(-0.00034)
|
|
|
|
|
|
| > avg_loss:[92m 13.30571 [0m(-0.02081)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93285 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_100.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 50/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:12:55) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0/2 -- GLOBAL_STEP: 100[0m
|
|
|
|
|
|
| > decoder_loss: 28.81597 (28.81597)
|
|
|
|
|
|
| > postnet_loss: 30.45031 (30.45031)
|
|
|
|
|
|
| > stopnet_loss: 0.77294 (0.77294)
|
|
|
|
|
|
| > loss: 15.58951 (15.58951)
|
|
|
|
|
|
| > align_error: 0.95763 (0.95763)
|
|
|
|
|
|
| > grad_norm: 7.40918 (7.40918)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 0.94340 (0.94343)
|
|
|
|
|
|
| > loader_time: 1.95140 (1.95145)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.42372 (25.42372)
|
|
|
|
|
|
| > postnet_loss: 24.52972 (24.52972)
|
|
|
|
|
|
| > stopnet_loss: 0.79416 (0.79416)
|
|
|
|
|
|
| > loss: 13.28252 (13.28252)
|
|
|
|
|
|
| > align_error: 0.93286 (0.93286)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.50761 [0m(-0.03927)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.42372 [0m(-0.00367)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.52972 [0m(-0.08833)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79416 [0m(-0.00019)
|
|
|
|
|
|
| > avg_loss:[92m 13.28252 [0m(-0.02319)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93286 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_102.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 51/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:13:12) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.41935 (25.41935)
|
|
|
|
|
|
| > postnet_loss: 24.44258 (24.44258)
|
|
|
|
|
|
| > stopnet_loss: 0.79388 (0.79388)
|
|
|
|
|
|
| > loss: 13.25936 (13.25936)
|
|
|
|
|
|
| > align_error: 0.93286 (0.93286)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.71373 [0m(+0.20612)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.41935 [0m(-0.00438)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.44258 [0m(-0.08713)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79388 [0m(-0.00028)
|
|
|
|
|
|
| > avg_loss:[92m 13.25936 [0m(-0.02315)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93286 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_104.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 52/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:13:41) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.41561 (25.41561)
|
|
|
|
|
|
| > postnet_loss: 24.36353 (24.36353)
|
|
|
|
|
|
| > stopnet_loss: 0.79354 (0.79354)
|
|
|
|
|
|
| > loss: 13.23832 (13.23832)
|
|
|
|
|
|
| > align_error: 0.93287 (0.93287)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.37132 [0m(-0.34242)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.41561 [0m(-0.00373)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.36353 [0m(-0.07906)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79354 [0m(-0.00035)
|
|
|
|
|
|
| > avg_loss:[92m 13.23832 [0m(-0.02104)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93287 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_106.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 53/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:13:56) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.41186 (25.41186)
|
|
|
|
|
|
| > postnet_loss: 24.27541 (24.27541)
|
|
|
|
|
|
| > stopnet_loss: 0.79319 (0.79319)
|
|
|
|
|
|
| > loss: 13.21501 (13.21501)
|
|
|
|
|
|
| > align_error: 0.93287 (0.93287)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.39573 [0m(+0.02441)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.41186 [0m(-0.00376)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.27541 [0m(-0.08812)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79319 [0m(-0.00034)
|
|
|
|
|
|
| > avg_loss:[92m 13.21501 [0m(-0.02331)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93287 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_108.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 54/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:14:12) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.40795 (25.40795)
|
|
|
|
|
|
| > postnet_loss: 24.18665 (24.18665)
|
|
|
|
|
|
| > stopnet_loss: 0.79294 (0.79294)
|
|
|
|
|
|
| > loss: 13.19159 (13.19159)
|
|
|
|
|
|
| > align_error: 0.93288 (0.93288)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.41590 [0m(+0.02017)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.40795 [0m(-0.00391)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.18665 [0m(-0.08875)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79294 [0m(-0.00026)
|
|
|
|
|
|
| > avg_loss:[92m 13.19159 [0m(-0.02342)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93288 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_110.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 55/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:14:28) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.40349 (25.40349)
|
|
|
|
|
|
| > postnet_loss: 24.09873 (24.09873)
|
|
|
|
|
|
| > stopnet_loss: 0.79259 (0.79259)
|
|
|
|
|
|
| > loss: 13.16814 (13.16814)
|
|
|
|
|
|
| > align_error: 0.93289 (0.93289)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.54688 [0m(+0.13098)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.40349 [0m(-0.00446)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.09873 [0m(-0.08793)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79259 [0m(-0.00035)
|
|
|
|
|
|
| > avg_loss:[92m 13.16814 [0m(-0.02345)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93289 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_112.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 56/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:14:44) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.39905 (25.39905)
|
|
|
|
|
|
| > postnet_loss: 24.00666 (24.00666)
|
|
|
|
|
|
| > stopnet_loss: 0.79248 (0.79248)
|
|
|
|
|
|
| > loss: 13.14391 (13.14391)
|
|
|
|
|
|
| > align_error: 0.93290 (0.93290)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.54733 [0m(+0.00045)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.39905 [0m(-0.00444)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 24.00666 [0m(-0.09206)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79248 [0m(-0.00011)
|
|
|
|
|
|
| > avg_loss:[92m 13.14391 [0m(-0.02423)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93290 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_114.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 57/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:14:59) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.39442 (25.39442)
|
|
|
|
|
|
| > postnet_loss: 23.90174 (23.90174)
|
|
|
|
|
|
| > stopnet_loss: 0.79226 (0.79226)
|
|
|
|
|
|
| > loss: 13.11630 (13.11630)
|
|
|
|
|
|
| > align_error: 0.93290 (0.93290)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.53197 [0m(-0.01536)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.39442 [0m(-0.00463)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 23.90174 [0m(-0.10492)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79226 [0m(-0.00022)
|
|
|
|
|
|
| > avg_loss:[92m 13.11630 [0m(-0.02761)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93290 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_116.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 58/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:15:14) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.38974 (25.38974)
|
|
|
|
|
|
| > postnet_loss: 23.79935 (23.79935)
|
|
|
|
|
|
| > stopnet_loss: 0.79199 (0.79199)
|
|
|
|
|
|
| > loss: 13.08926 (13.08926)
|
|
|
|
|
|
| > align_error: 0.93291 (0.93291)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.56064 [0m(+0.02867)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.38974 [0m(-0.00468)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 23.79935 [0m(-0.10240)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79199 [0m(-0.00027)
|
|
|
|
|
|
| > avg_loss:[92m 13.08926 [0m(-0.02704)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93291 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_118.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 59/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:15:29) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.38513 (25.38513)
|
|
|
|
|
|
| > postnet_loss: 23.68726 (23.68726)
|
|
|
|
|
|
| > stopnet_loss: 0.79160 (0.79160)
|
|
|
|
|
|
| > loss: 13.05970 (13.05970)
|
|
|
|
|
|
| > align_error: 0.93292 (0.93292)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.09903 [0m(+0.53839)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.38513 [0m(-0.00461)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 23.68726 [0m(-0.11209)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79160 [0m(-0.00039)
|
|
|
|
|
|
| > avg_loss:[92m 13.05970 [0m(-0.02957)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93292 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_120.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 60/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:15:58) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.38030 (25.38030)
|
|
|
|
|
|
| > postnet_loss: 23.58084 (23.58084)
|
|
|
|
|
|
| > stopnet_loss: 0.79127 (0.79127)
|
|
|
|
|
|
| > loss: 13.03155 (13.03155)
|
|
|
|
|
|
| > align_error: 0.93293 (0.93293)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.39232 [0m(-0.70671)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.38030 [0m(-0.00483)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 23.58084 [0m(-0.10642)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79127 [0m(-0.00033)
|
|
|
|
|
|
| > avg_loss:[92m 13.03155 [0m(-0.02814)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93293 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_122.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 61/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:16:14) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.37555 (25.37555)
|
|
|
|
|
|
| > postnet_loss: 23.46654 (23.46654)
|
|
|
|
|
|
| > stopnet_loss: 0.79107 (0.79107)
|
|
|
|
|
|
| > loss: 13.00159 (13.00159)
|
|
|
|
|
|
| > align_error: 0.93294 (0.93294)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.45233 [0m(+0.06001)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.37555 [0m(-0.00475)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 23.46654 [0m(-0.11430)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79107 [0m(-0.00020)
|
|
|
|
|
|
| > avg_loss:[92m 13.00159 [0m(-0.02996)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93294 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_124.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 62/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:16:30) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 1/2 -- GLOBAL_STEP: 125[0m
|
|
|
|
|
|
| > decoder_loss: 30.75975 (30.75975)
|
|
|
|
|
|
| > postnet_loss: 31.73096 (31.73096)
|
|
|
|
|
|
| > stopnet_loss: 0.78231 (0.78231)
|
|
|
|
|
|
| > loss: 16.40499 (16.40499)
|
|
|
|
|
|
| > align_error: 0.96980 (0.96980)
|
|
|
|
|
|
| > grad_norm: 9.25334 (9.25334)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 2.29630 (2.29632)
|
|
|
|
|
|
| > loader_time: 0.00520 (0.00517)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.37055 (25.37055)
|
|
|
|
|
|
| > postnet_loss: 23.36364 (23.36364)
|
|
|
|
|
|
| > stopnet_loss: 0.79070 (0.79070)
|
|
|
|
|
|
| > loss: 12.97425 (12.97425)
|
|
|
|
|
|
| > align_error: 0.93295 (0.93295)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.40448 [0m(-0.04784)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.37055 [0m(-0.00500)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 23.36364 [0m(-0.10289)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79070 [0m(-0.00037)
|
|
|
|
|
|
| > avg_loss:[92m 12.97425 [0m(-0.02735)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93295 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_126.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 63/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:16:46) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.36542 (25.36542)
|
|
|
|
|
|
| > postnet_loss: 23.24089 (23.24089)
|
|
|
|
|
|
| > stopnet_loss: 0.79048 (0.79048)
|
|
|
|
|
|
| > loss: 12.94206 (12.94206)
|
|
|
|
|
|
| > align_error: 0.93296 (0.93296)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.43120 [0m(+0.02672)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.36542 [0m(-0.00513)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 23.24089 [0m(-0.12275)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79048 [0m(-0.00022)
|
|
|
|
|
|
| > avg_loss:[92m 12.94206 [0m(-0.03219)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93296 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_128.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 64/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:17:02) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.36027 (25.36027)
|
|
|
|
|
|
| > postnet_loss: 23.12474 (23.12474)
|
|
|
|
|
|
| > stopnet_loss: 0.79020 (0.79020)
|
|
|
|
|
|
| > loss: 12.91146 (12.91146)
|
|
|
|
|
|
| > align_error: 0.93297 (0.93297)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.38159 [0m(-0.04961)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.36027 [0m(-0.00516)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 23.12474 [0m(-0.11615)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79020 [0m(-0.00027)
|
|
|
|
|
|
| > avg_loss:[92m 12.91146 [0m(-0.03060)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93297 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_130.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 65/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:17:17) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.35475 (25.35475)
|
|
|
|
|
|
| > postnet_loss: 23.00372 (23.00372)
|
|
|
|
|
|
| > stopnet_loss: 0.79002 (0.79002)
|
|
|
|
|
|
| > loss: 12.87964 (12.87964)
|
|
|
|
|
|
| > align_error: 0.93298 (0.93298)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.36091 [0m(-0.02068)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.35475 [0m(-0.00551)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 23.00372 [0m(-0.12102)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.79002 [0m(-0.00019)
|
|
|
|
|
|
| > avg_loss:[92m 12.87964 [0m(-0.03182)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93298 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_132.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 66/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:17:34) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.34929 (25.34929)
|
|
|
|
|
|
| > postnet_loss: 22.88660 (22.88660)
|
|
|
|
|
|
| > stopnet_loss: 0.78961 (0.78961)
|
|
|
|
|
|
| > loss: 12.84858 (12.84858)
|
|
|
|
|
|
| > align_error: 0.93300 (0.93300)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.53354 [0m(+0.17264)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.34929 [0m(-0.00547)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 22.88660 [0m(-0.11712)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78961 [0m(-0.00041)
|
|
|
|
|
|
| > avg_loss:[92m 12.84858 [0m(-0.03106)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93300 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_134.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 67/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:17:50) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.34374 (25.34374)
|
|
|
|
|
|
| > postnet_loss: 22.75751 (22.75751)
|
|
|
|
|
|
| > stopnet_loss: 0.78918 (0.78918)
|
|
|
|
|
|
| > loss: 12.81449 (12.81449)
|
|
|
|
|
|
| > align_error: 0.93301 (0.93301)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.53269 [0m(-0.00085)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.34374 [0m(-0.00554)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 22.75751 [0m(-0.12910)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78918 [0m(-0.00043)
|
|
|
|
|
|
| > avg_loss:[92m 12.81449 [0m(-0.03409)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93301 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_136.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 68/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:18:14) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.33797 (25.33797)
|
|
|
|
|
|
| > postnet_loss: 22.62930 (22.62930)
|
|
|
|
|
|
| > stopnet_loss: 0.78872 (0.78872)
|
|
|
|
|
|
| > loss: 12.78054 (12.78054)
|
|
|
|
|
|
| > align_error: 0.93302 (0.93302)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.41216 [0m(-0.12054)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.33797 [0m(-0.00578)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 22.62930 [0m(-0.12821)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78872 [0m(-0.00046)
|
|
|
|
|
|
| > avg_loss:[92m 12.78054 [0m(-0.03395)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93302 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_138.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 69/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:18:35) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.33159 (25.33159)
|
|
|
|
|
|
| > postnet_loss: 22.48619 (22.48619)
|
|
|
|
|
|
| > stopnet_loss: 0.78840 (0.78840)
|
|
|
|
|
|
| > loss: 12.74284 (12.74284)
|
|
|
|
|
|
| > align_error: 0.93303 (0.93303)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.36412 [0m(-0.04803)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.33159 [0m(-0.00637)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 22.48619 [0m(-0.14311)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78840 [0m(-0.00033)
|
|
|
|
|
|
| > avg_loss:[92m 12.74284 [0m(-0.03770)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93303 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_140.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 70/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:18:50) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.32540 (25.32540)
|
|
|
|
|
|
| > postnet_loss: 22.35530 (22.35530)
|
|
|
|
|
|
| > stopnet_loss: 0.78784 (0.78784)
|
|
|
|
|
|
| > loss: 12.70801 (12.70801)
|
|
|
|
|
|
| > align_error: 0.93304 (0.93304)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.36249 [0m(-0.00163)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.32540 [0m(-0.00620)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 22.35530 [0m(-0.13089)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78784 [0m(-0.00056)
|
|
|
|
|
|
| > avg_loss:[92m 12.70801 [0m(-0.03483)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93304 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_142.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 71/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:19:05) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.31923 (25.31923)
|
|
|
|
|
|
| > postnet_loss: 22.21478 (22.21478)
|
|
|
|
|
|
| > stopnet_loss: 0.78753 (0.78753)
|
|
|
|
|
|
| > loss: 12.67104 (12.67104)
|
|
|
|
|
|
| > align_error: 0.93306 (0.93306)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.35915 [0m(-0.00334)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.31923 [0m(-0.00617)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 22.21478 [0m(-0.14051)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78753 [0m(-0.00030)
|
|
|
|
|
|
| > avg_loss:[92m 12.67104 [0m(-0.03697)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93306 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_144.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 72/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:19:21) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.31329 (25.31329)
|
|
|
|
|
|
| > postnet_loss: 22.07190 (22.07190)
|
|
|
|
|
|
| > stopnet_loss: 0.78725 (0.78725)
|
|
|
|
|
|
| > loss: 12.63354 (12.63354)
|
|
|
|
|
|
| > align_error: 0.93307 (0.93307)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.36407 [0m(+0.00492)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.31329 [0m(-0.00594)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 22.07190 [0m(-0.14288)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78725 [0m(-0.00029)
|
|
|
|
|
|
| > avg_loss:[92m 12.63354 [0m(-0.03749)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93307 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_146.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 73/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:19:36) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.30707 (25.30707)
|
|
|
|
|
|
| > postnet_loss: 21.92090 (21.92090)
|
|
|
|
|
|
| > stopnet_loss: 0.78672 (0.78672)
|
|
|
|
|
|
| > loss: 12.59372 (12.59372)
|
|
|
|
|
|
| > align_error: 0.93308 (0.93308)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.40378 [0m(+0.03971)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.30707 [0m(-0.00622)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 21.92090 [0m(-0.15100)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78672 [0m(-0.00052)
|
|
|
|
|
|
| > avg_loss:[92m 12.59372 [0m(-0.03983)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93308 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_148.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 74/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:19:51) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.30068 (25.30068)
|
|
|
|
|
|
| > postnet_loss: 21.78074 (21.78074)
|
|
|
|
|
|
| > stopnet_loss: 0.78630 (0.78630)
|
|
|
|
|
|
| > loss: 12.55666 (12.55666)
|
|
|
|
|
|
| > align_error: 0.93310 (0.93310)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.39449 [0m(-0.00929)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.30068 [0m(-0.00638)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 21.78074 [0m(-0.14016)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78630 [0m(-0.00042)
|
|
|
|
|
|
| > avg_loss:[92m 12.55666 [0m(-0.03706)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93310 [0m(+0.00001)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_150.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 75/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:20:06) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0/2 -- GLOBAL_STEP: 150[0m
|
|
|
|
|
|
| > decoder_loss: 28.70810 (28.70810)
|
|
|
|
|
|
| > postnet_loss: 29.17132 (29.17132)
|
|
|
|
|
|
| > stopnet_loss: 0.76647 (0.76647)
|
|
|
|
|
|
| > loss: 15.23632 (15.23632)
|
|
|
|
|
|
| > align_error: 0.95765 (0.95765)
|
|
|
|
|
|
| > grad_norm: 7.57412 (7.57412)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 0.77170 (0.77172)
|
|
|
|
|
|
| > loader_time: 1.60250 (1.60249)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.29371 (25.29371)
|
|
|
|
|
|
| > postnet_loss: 21.61256 (21.61256)
|
|
|
|
|
|
| > stopnet_loss: 0.78598 (0.78598)
|
|
|
|
|
|
| > loss: 12.51255 (12.51255)
|
|
|
|
|
|
| > align_error: 0.93311 (0.93311)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.35683 [0m(-0.03766)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.29371 [0m(-0.00697)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 21.61256 [0m(-0.16819)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78598 [0m(-0.00032)
|
|
|
|
|
|
| > avg_loss:[92m 12.51255 [0m(-0.04411)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93311 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_152.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 76/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:20:27) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.28675 (25.28675)
|
|
|
|
|
|
| > postnet_loss: 21.46666 (21.46666)
|
|
|
|
|
|
| > stopnet_loss: 0.78554 (0.78554)
|
|
|
|
|
|
| > loss: 12.47389 (12.47389)
|
|
|
|
|
|
| > align_error: 0.93313 (0.93313)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.27413 [0m(+0.91730)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.28675 [0m(-0.00697)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 21.46666 [0m(-0.14590)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78554 [0m(-0.00044)
|
|
|
|
|
|
| > avg_loss:[92m 12.47389 [0m(-0.03865)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93313 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_154.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 77/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:20:51) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.28025 (25.28025)
|
|
|
|
|
|
| > postnet_loss: 21.31206 (21.31206)
|
|
|
|
|
|
| > stopnet_loss: 0.78529 (0.78529)
|
|
|
|
|
|
| > loss: 12.43336 (12.43336)
|
|
|
|
|
|
| > align_error: 0.93314 (0.93314)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.51127 [0m(-0.76286)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.28025 [0m(-0.00650)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 21.31206 [0m(-0.15460)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78529 [0m(-0.00025)
|
|
|
|
|
|
| > avg_loss:[92m 12.43336 [0m(-0.04053)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93314 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_156.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 78/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:21:06) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.27306 (25.27306)
|
|
|
|
|
|
| > postnet_loss: 21.16508 (21.16508)
|
|
|
|
|
|
| > stopnet_loss: 0.78484 (0.78484)
|
|
|
|
|
|
| > loss: 12.39437 (12.39437)
|
|
|
|
|
|
| > align_error: 0.93316 (0.93316)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.52373 [0m(+0.01247)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.27306 [0m(-0.00719)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 21.16508 [0m(-0.14698)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78484 [0m(-0.00045)
|
|
|
|
|
|
| > avg_loss:[92m 12.39437 [0m(-0.03899)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93316 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_158.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 79/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:21:20) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.26594 (25.26594)
|
|
|
|
|
|
| > postnet_loss: 21.00286 (21.00286)
|
|
|
|
|
|
| > stopnet_loss: 0.78445 (0.78445)
|
|
|
|
|
|
| > loss: 12.35165 (12.35165)
|
|
|
|
|
|
| > align_error: 0.93317 (0.93317)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.55061 [0m(+0.02688)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.26594 [0m(-0.00712)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 21.00286 [0m(-0.16222)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78445 [0m(-0.00038)
|
|
|
|
|
|
| > avg_loss:[92m 12.35165 [0m(-0.04272)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93317 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_160.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 80/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:21:35) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.25847 (25.25847)
|
|
|
|
|
|
| > postnet_loss: 20.82819 (20.82819)
|
|
|
|
|
|
| > stopnet_loss: 0.78421 (0.78421)
|
|
|
|
|
|
| > loss: 12.30588 (12.30588)
|
|
|
|
|
|
| > align_error: 0.93319 (0.93319)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.52145 [0m(-0.02916)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.25847 [0m(-0.00747)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 20.82819 [0m(-0.17467)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78421 [0m(-0.00024)
|
|
|
|
|
|
| > avg_loss:[92m 12.30588 [0m(-0.04578)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93319 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_162.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 81/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:21:51) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.25085 (25.25085)
|
|
|
|
|
|
| > postnet_loss: 20.66319 (20.66319)
|
|
|
|
|
|
| > stopnet_loss: 0.78364 (0.78364)
|
|
|
|
|
|
| > loss: 12.26215 (12.26215)
|
|
|
|
|
|
| > align_error: 0.93321 (0.93321)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.56148 [0m(+0.04003)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.25085 [0m(-0.00762)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 20.66319 [0m(-0.16501)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78364 [0m(-0.00057)
|
|
|
|
|
|
| > avg_loss:[92m 12.26215 [0m(-0.04373)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93321 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_164.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 82/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:22:06) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.24309 (25.24309)
|
|
|
|
|
|
| > postnet_loss: 20.50424 (20.50424)
|
|
|
|
|
|
| > stopnet_loss: 0.78318 (0.78318)
|
|
|
|
|
|
| > loss: 12.22001 (12.22001)
|
|
|
|
|
|
| > align_error: 0.93322 (0.93322)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.53557 [0m(-0.02591)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.24309 [0m(-0.00776)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 20.50424 [0m(-0.15894)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78318 [0m(-0.00046)
|
|
|
|
|
|
| > avg_loss:[92m 12.22001 [0m(-0.04214)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93322 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_166.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 83/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:22:22) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.23509 (25.23509)
|
|
|
|
|
|
| > postnet_loss: 20.33735 (20.33735)
|
|
|
|
|
|
| > stopnet_loss: 0.78274 (0.78274)
|
|
|
|
|
|
| > loss: 12.17585 (12.17585)
|
|
|
|
|
|
| > align_error: 0.93324 (0.93324)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.55396 [0m(+0.01839)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.23509 [0m(-0.00800)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 20.33735 [0m(-0.16689)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78274 [0m(-0.00044)
|
|
|
|
|
|
| > avg_loss:[92m 12.17585 [0m(-0.04416)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93324 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_168.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 84/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:22:38) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.22756 (25.22756)
|
|
|
|
|
|
| > postnet_loss: 20.16981 (20.16981)
|
|
|
|
|
|
| > stopnet_loss: 0.78234 (0.78234)
|
|
|
|
|
|
| > loss: 12.13168 (12.13168)
|
|
|
|
|
|
| > align_error: 0.93326 (0.93326)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.78248 [0m(+0.22852)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.22756 [0m(-0.00752)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 20.16981 [0m(-0.16754)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78234 [0m(-0.00040)
|
|
|
|
|
|
| > avg_loss:[92m 12.13168 [0m(-0.04417)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93326 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_170.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 85/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:23:08) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.21922 (25.21922)
|
|
|
|
|
|
| > postnet_loss: 19.99962 (19.99962)
|
|
|
|
|
|
| > stopnet_loss: 0.78191 (0.78191)
|
|
|
|
|
|
| > loss: 12.08662 (12.08662)
|
|
|
|
|
|
| > align_error: 0.93327 (0.93327)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.40640 [0m(-0.37608)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.21922 [0m(-0.00834)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 19.99962 [0m(-0.17019)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78191 [0m(-0.00043)
|
|
|
|
|
|
| > avg_loss:[92m 12.08662 [0m(-0.04506)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93327 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_172.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 86/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:23:23) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.21048 (25.21048)
|
|
|
|
|
|
| > postnet_loss: 19.82795 (19.82795)
|
|
|
|
|
|
| > stopnet_loss: 0.78128 (0.78128)
|
|
|
|
|
|
| > loss: 12.04089 (12.04089)
|
|
|
|
|
|
| > align_error: 0.93329 (0.93329)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.40720 [0m(+0.00080)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.21048 [0m(-0.00874)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 19.82795 [0m(-0.17167)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78128 [0m(-0.00062)
|
|
|
|
|
|
| > avg_loss:[92m 12.04089 [0m(-0.04572)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93329 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_174.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 87/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:23:39) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 1/2 -- GLOBAL_STEP: 175[0m
|
|
|
|
|
|
| > decoder_loss: 30.60882 (30.60882)
|
|
|
|
|
|
| > postnet_loss: 29.56367 (29.56367)
|
|
|
|
|
|
| > stopnet_loss: 0.77738 (0.77738)
|
|
|
|
|
|
| > loss: 15.82050 (15.82050)
|
|
|
|
|
|
| > align_error: 0.96989 (0.96989)
|
|
|
|
|
|
| > grad_norm: 9.53094 (9.53094)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 2.32800 (2.32796)
|
|
|
|
|
|
| > loader_time: 0.00510 (0.00514)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.20211 (25.20211)
|
|
|
|
|
|
| > postnet_loss: 19.65288 (19.65288)
|
|
|
|
|
|
| > stopnet_loss: 0.78097 (0.78097)
|
|
|
|
|
|
| > loss: 11.99471 (11.99471)
|
|
|
|
|
|
| > align_error: 0.93331 (0.93331)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.53988 [0m(+0.13269)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.20211 [0m(-0.00838)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 19.65288 [0m(-0.17507)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78097 [0m(-0.00032)
|
|
|
|
|
|
| > avg_loss:[92m 11.99471 [0m(-0.04618)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93331 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_176.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 88/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:23:55) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.19364 (25.19364)
|
|
|
|
|
|
| > postnet_loss: 19.48893 (19.48893)
|
|
|
|
|
|
| > stopnet_loss: 0.78055 (0.78055)
|
|
|
|
|
|
| > loss: 11.95120 (11.95120)
|
|
|
|
|
|
| > align_error: 0.93333 (0.93333)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.58129 [0m(+0.04141)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.19364 [0m(-0.00846)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 19.48893 [0m(-0.16395)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78055 [0m(-0.00041)
|
|
|
|
|
|
| > avg_loss:[92m 11.95120 [0m(-0.04352)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93333 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_178.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 89/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:24:11) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.18482 (25.18482)
|
|
|
|
|
|
| > postnet_loss: 19.31188 (19.31188)
|
|
|
|
|
|
| > stopnet_loss: 0.78017 (0.78017)
|
|
|
|
|
|
| > loss: 11.90434 (11.90434)
|
|
|
|
|
|
| > align_error: 0.93335 (0.93335)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.58293 [0m(+0.00164)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.18482 [0m(-0.00883)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 19.31188 [0m(-0.17705)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.78017 [0m(-0.00038)
|
|
|
|
|
|
| > avg_loss:[92m 11.90434 [0m(-0.04685)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93335 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_180.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 90/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:24:27) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.17611 (25.17611)
|
|
|
|
|
|
| > postnet_loss: 19.12614 (19.12614)
|
|
|
|
|
|
| > stopnet_loss: 0.77964 (0.77964)
|
|
|
|
|
|
| > loss: 11.85520 (11.85520)
|
|
|
|
|
|
| > align_error: 0.93337 (0.93337)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.54274 [0m(-0.04020)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.17611 [0m(-0.00871)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 19.12614 [0m(-0.18574)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77964 [0m(-0.00053)
|
|
|
|
|
|
| > avg_loss:[92m 11.85520 [0m(-0.04914)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93337 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_182.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 91/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:24:42) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.16712 (25.16712)
|
|
|
|
|
|
| > postnet_loss: 18.94039 (18.94039)
|
|
|
|
|
|
| > stopnet_loss: 0.77921 (0.77921)
|
|
|
|
|
|
| > loss: 11.80609 (11.80609)
|
|
|
|
|
|
| > align_error: 0.93339 (0.93339)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.52540 [0m(-0.01734)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.16712 [0m(-0.00899)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 18.94039 [0m(-0.18575)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77921 [0m(-0.00044)
|
|
|
|
|
|
| > avg_loss:[92m 11.80609 [0m(-0.04912)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93339 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_184.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 92/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:24:57) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.15761 (25.15761)
|
|
|
|
|
|
| > postnet_loss: 18.76252 (18.76252)
|
|
|
|
|
|
| > stopnet_loss: 0.77867 (0.77867)
|
|
|
|
|
|
| > loss: 11.75870 (11.75870)
|
|
|
|
|
|
| > align_error: 0.93341 (0.93341)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.30599 [0m(+0.78059)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.15761 [0m(-0.00951)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 18.76252 [0m(-0.17788)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77867 [0m(-0.00054)
|
|
|
|
|
|
| > avg_loss:[92m 11.75870 [0m(-0.04738)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93341 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_186.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 93/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:25:25) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.14812 (25.14812)
|
|
|
|
|
|
| > postnet_loss: 18.58510 (18.58510)
|
|
|
|
|
|
| > stopnet_loss: 0.77830 (0.77830)
|
|
|
|
|
|
| > loss: 11.71161 (11.71161)
|
|
|
|
|
|
| > align_error: 0.93343 (0.93343)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.39246 [0m(-0.91353)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.14812 [0m(-0.00949)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 18.58510 [0m(-0.17742)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77830 [0m(-0.00037)
|
|
|
|
|
|
| > avg_loss:[92m 11.71161 [0m(-0.04709)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93343 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_188.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 94/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:25:43) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.13870 (25.13870)
|
|
|
|
|
|
| > postnet_loss: 18.39444 (18.39444)
|
|
|
|
|
|
| > stopnet_loss: 0.77788 (0.77788)
|
|
|
|
|
|
| > loss: 11.66116 (11.66116)
|
|
|
|
|
|
| > align_error: 0.93345 (0.93345)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.39401 [0m(+0.00155)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.13870 [0m(-0.00942)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 18.39444 [0m(-0.19066)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77788 [0m(-0.00042)
|
|
|
|
|
|
| > avg_loss:[92m 11.66116 [0m(-0.05044)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93345 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_190.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 95/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:25:59) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.12885 (25.12885)
|
|
|
|
|
|
| > postnet_loss: 18.21178 (18.21178)
|
|
|
|
|
|
| > stopnet_loss: 0.77736 (0.77736)
|
|
|
|
|
|
| > loss: 11.61252 (11.61252)
|
|
|
|
|
|
| > align_error: 0.93347 (0.93347)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.39205 [0m(-0.00196)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.12885 [0m(-0.00985)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 18.21178 [0m(-0.18266)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77736 [0m(-0.00052)
|
|
|
|
|
|
| > avg_loss:[92m 11.61252 [0m(-0.04865)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93347 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_192.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 96/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:26:14) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.11837 (25.11837)
|
|
|
|
|
|
| > postnet_loss: 18.02538 (18.02538)
|
|
|
|
|
|
| > stopnet_loss: 0.77653 (0.77653)
|
|
|
|
|
|
| > loss: 11.56247 (11.56247)
|
|
|
|
|
|
| > align_error: 0.93349 (0.93349)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.38313 [0m(-0.00892)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.11837 [0m(-0.01048)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 18.02538 [0m(-0.18640)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77653 [0m(-0.00083)
|
|
|
|
|
|
| > avg_loss:[92m 11.56247 [0m(-0.05005)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93349 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_194.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 97/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:26:30) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.10795 (25.10795)
|
|
|
|
|
|
| > postnet_loss: 17.84328 (17.84328)
|
|
|
|
|
|
| > stopnet_loss: 0.77577 (0.77577)
|
|
|
|
|
|
| > loss: 11.51358 (11.51358)
|
|
|
|
|
|
| > align_error: 0.93352 (0.93352)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.40798 [0m(+0.02485)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.10795 [0m(-0.01042)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 17.84328 [0m(-0.18210)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77577 [0m(-0.00076)
|
|
|
|
|
|
| > avg_loss:[92m 11.51358 [0m(-0.04889)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93352 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_196.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 98/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:26:46) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.09743 (25.09743)
|
|
|
|
|
|
| > postnet_loss: 17.66113 (17.66113)
|
|
|
|
|
|
| > stopnet_loss: 0.77518 (0.77518)
|
|
|
|
|
|
| > loss: 11.46482 (11.46482)
|
|
|
|
|
|
| > align_error: 0.93354 (0.93354)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.53481 [0m(+0.12683)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.09743 [0m(-0.01052)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 17.66113 [0m(-0.18215)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77518 [0m(-0.00059)
|
|
|
|
|
|
| > avg_loss:[92m 11.46482 [0m(-0.04876)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93354 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_198.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 99/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:27:02) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.08615 (25.08615)
|
|
|
|
|
|
| > postnet_loss: 17.48265 (17.48265)
|
|
|
|
|
|
| > stopnet_loss: 0.77459 (0.77459)
|
|
|
|
|
|
| > loss: 11.41679 (11.41679)
|
|
|
|
|
|
| > align_error: 0.93356 (0.93356)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.57076 [0m(+0.03595)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.08615 [0m(-0.01129)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 17.48265 [0m(-0.17848)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77459 [0m(-0.00059)
|
|
|
|
|
|
| > avg_loss:[92m 11.41679 [0m(-0.04803)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93356 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_200.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 100/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:27:17) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0/2 -- GLOBAL_STEP: 200[0m
|
|
|
|
|
|
| > decoder_loss: 28.51859 (28.51859)
|
|
|
|
|
|
| > postnet_loss: 26.93233 (26.93233)
|
|
|
|
|
|
| > stopnet_loss: 0.75555 (0.75555)
|
|
|
|
|
|
| > loss: 14.61827 (14.61827)
|
|
|
|
|
|
| > align_error: 0.95778 (0.95778)
|
|
|
|
|
|
| > grad_norm: 7.73923 (7.73923)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 0.86440 (0.86444)
|
|
|
|
|
|
| > loader_time: 1.88740 (1.88742)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.07503 (25.07503)
|
|
|
|
|
|
| > postnet_loss: 17.30268 (17.30268)
|
|
|
|
|
|
| > stopnet_loss: 0.77407 (0.77407)
|
|
|
|
|
|
| > loss: 11.36849 (11.36849)
|
|
|
|
|
|
| > align_error: 0.93358 (0.93358)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.56897 [0m(-0.00178)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.07503 [0m(-0.01112)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 17.30268 [0m(-0.17997)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77407 [0m(-0.00052)
|
|
|
|
|
|
| > avg_loss:[92m 11.36849 [0m(-0.04829)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93358 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_202.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 101/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:27:41) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.06417 (25.06417)
|
|
|
|
|
|
| > postnet_loss: 17.11801 (17.11801)
|
|
|
|
|
|
| > stopnet_loss: 0.77357 (0.77357)
|
|
|
|
|
|
| > loss: 11.31911 (11.31911)
|
|
|
|
|
|
| > align_error: 0.93361 (0.93361)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.40333 [0m(-0.16564)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.06417 [0m(-0.01086)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 17.11801 [0m(-0.18467)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77357 [0m(-0.00050)
|
|
|
|
|
|
| > avg_loss:[92m 11.31911 [0m(-0.04938)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93361 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_204.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 102/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:28:02) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.05266 (25.05266)
|
|
|
|
|
|
| > postnet_loss: 16.93483 (16.93483)
|
|
|
|
|
|
| > stopnet_loss: 0.77312 (0.77312)
|
|
|
|
|
|
| > loss: 11.26999 (11.26999)
|
|
|
|
|
|
| > align_error: 0.93363 (0.93363)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.38345 [0m(-0.01988)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.05266 [0m(-0.01151)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 16.93483 [0m(-0.18318)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77312 [0m(-0.00045)
|
|
|
|
|
|
| > avg_loss:[92m 11.26999 [0m(-0.04912)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93363 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_206.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 103/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:28:17) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.04104 (25.04104)
|
|
|
|
|
|
| > postnet_loss: 16.75883 (16.75883)
|
|
|
|
|
|
| > stopnet_loss: 0.77256 (0.77256)
|
|
|
|
|
|
| > loss: 11.22252 (11.22252)
|
|
|
|
|
|
| > align_error: 0.93366 (0.93366)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.39880 [0m(+0.01535)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.04104 [0m(-0.01162)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 16.75883 [0m(-0.17600)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77256 [0m(-0.00056)
|
|
|
|
|
|
| > avg_loss:[92m 11.22252 [0m(-0.04747)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93366 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_208.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 104/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:28:33) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.02906 (25.02906)
|
|
|
|
|
|
| > postnet_loss: 16.59436 (16.59436)
|
|
|
|
|
|
| > stopnet_loss: 0.77200 (0.77200)
|
|
|
|
|
|
| > loss: 11.17786 (11.17786)
|
|
|
|
|
|
| > align_error: 0.93368 (0.93368)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.39934 [0m(+0.00055)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.02906 [0m(-0.01199)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 16.59436 [0m(-0.16447)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77200 [0m(-0.00055)
|
|
|
|
|
|
| > avg_loss:[92m 11.17786 [0m(-0.04467)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93368 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_210.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 105/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:28:49) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.01605 (25.01605)
|
|
|
|
|
|
| > postnet_loss: 16.41172 (16.41172)
|
|
|
|
|
|
| > stopnet_loss: 0.77131 (0.77131)
|
|
|
|
|
|
| > loss: 11.12825 (11.12825)
|
|
|
|
|
|
| > align_error: 0.93371 (0.93371)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.38015 [0m(-0.01920)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.01605 [0m(-0.01301)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 16.41172 [0m(-0.18264)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77131 [0m(-0.00069)
|
|
|
|
|
|
| > avg_loss:[92m 11.12825 [0m(-0.04960)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93371 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_212.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 106/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:29:04) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 25.00313 (25.00313)
|
|
|
|
|
|
| > postnet_loss: 16.23350 (16.23350)
|
|
|
|
|
|
| > stopnet_loss: 0.77095 (0.77095)
|
|
|
|
|
|
| > loss: 11.08010 (11.08010)
|
|
|
|
|
|
| > align_error: 0.93373 (0.93373)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.39734 [0m(+0.01719)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 25.00313 [0m(-0.01292)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 16.23350 [0m(-0.17821)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77095 [0m(-0.00036)
|
|
|
|
|
|
| > avg_loss:[92m 11.08010 [0m(-0.04815)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93373 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_214.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 107/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:29:20) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.99050 (24.99050)
|
|
|
|
|
|
| > postnet_loss: 16.06552 (16.06552)
|
|
|
|
|
|
| > stopnet_loss: 0.77052 (0.77052)
|
|
|
|
|
|
| > loss: 11.03452 (11.03452)
|
|
|
|
|
|
| > align_error: 0.93375 (0.93375)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.39366 [0m(-0.00368)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.99050 [0m(-0.01263)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 16.06552 [0m(-0.16798)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77052 [0m(-0.00042)
|
|
|
|
|
|
| > avg_loss:[92m 11.03452 [0m(-0.04558)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93375 [0m(+0.00002)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_216.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 108/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:29:35) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.97730 (24.97730)
|
|
|
|
|
|
| > postnet_loss: 15.90222 (15.90222)
|
|
|
|
|
|
| > stopnet_loss: 0.77000 (0.77000)
|
|
|
|
|
|
| > loss: 10.98987 (10.98987)
|
|
|
|
|
|
| > align_error: 0.93378 (0.93378)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.38767 [0m(-0.00599)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.97730 [0m(-0.01320)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 15.90222 [0m(-0.16330)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.77000 [0m(-0.00053)
|
|
|
|
|
|
| > avg_loss:[92m 10.98987 [0m(-0.04465)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93378 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_218.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 109/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:29:57) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.96291 (24.96291)
|
|
|
|
|
|
| > postnet_loss: 15.73567 (15.73567)
|
|
|
|
|
|
| > stopnet_loss: 0.76924 (0.76924)
|
|
|
|
|
|
| > loss: 10.94389 (10.94389)
|
|
|
|
|
|
| > align_error: 0.93381 (0.93381)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.56949 [0m(+1.18181)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.96291 [0m(-0.01439)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 15.73567 [0m(-0.16655)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.76924 [0m(-0.00075)
|
|
|
|
|
|
| > avg_loss:[92m 10.94389 [0m(-0.04599)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93381 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_220.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 110/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:30:20) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.94934 (24.94934)
|
|
|
|
|
|
| > postnet_loss: 15.56663 (15.56663)
|
|
|
|
|
|
| > stopnet_loss: 0.76870 (0.76870)
|
|
|
|
|
|
| > loss: 10.89769 (10.89769)
|
|
|
|
|
|
| > align_error: 0.93383 (0.93383)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.40486 [0m(-1.16463)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.94934 [0m(-0.01357)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 15.56663 [0m(-0.16904)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.76870 [0m(-0.00054)
|
|
|
|
|
|
| > avg_loss:[92m 10.89769 [0m(-0.04620)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93383 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_222.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 111/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:30:36) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.93512 (24.93512)
|
|
|
|
|
|
| > postnet_loss: 15.41228 (15.41228)
|
|
|
|
|
|
| > stopnet_loss: 0.76803 (0.76803)
|
|
|
|
|
|
| > loss: 10.85488 (10.85488)
|
|
|
|
|
|
| > align_error: 0.93386 (0.93386)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.38025 [0m(-0.02460)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.93512 [0m(-0.01423)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 15.41228 [0m(-0.15435)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.76803 [0m(-0.00067)
|
|
|
|
|
|
| > avg_loss:[92m 10.85488 [0m(-0.04281)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93386 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_224.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 112/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:30:52) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 1/2 -- GLOBAL_STEP: 225[0m
|
|
|
|
|
|
| > decoder_loss: 30.33993 (30.33993)
|
|
|
|
|
|
| > postnet_loss: 26.17014 (26.17014)
|
|
|
|
|
|
| > stopnet_loss: 0.76161 (0.76161)
|
|
|
|
|
|
| > loss: 14.88913 (14.88913)
|
|
|
|
|
|
| > align_error: 0.97002 (0.97002)
|
|
|
|
|
|
| > grad_norm: 8.83697 (8.83697)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 2.75190 (2.75195)
|
|
|
|
|
|
| > loader_time: 0.00990 (0.00987)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.91999 (24.91999)
|
|
|
|
|
|
| > postnet_loss: 15.25691 (15.25691)
|
|
|
|
|
|
| > stopnet_loss: 0.76717 (0.76717)
|
|
|
|
|
|
| > loss: 10.81139 (10.81139)
|
|
|
|
|
|
| > align_error: 0.93388 (0.93388)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.39298 [0m(+0.01273)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.91999 [0m(-0.01512)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 15.25691 [0m(-0.15536)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.76717 [0m(-0.00087)
|
|
|
|
|
|
| > avg_loss:[92m 10.81139 [0m(-0.04349)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93388 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_226.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 113/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:31:07) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.90399 (24.90399)
|
|
|
|
|
|
| > postnet_loss: 15.09707 (15.09707)
|
|
|
|
|
|
| > stopnet_loss: 0.76637 (0.76637)
|
|
|
|
|
|
| > loss: 10.76663 (10.76663)
|
|
|
|
|
|
| > align_error: 0.93391 (0.93391)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.38124 [0m(-0.01174)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.90399 [0m(-0.01600)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 15.09707 [0m(-0.15985)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.76637 [0m(-0.00080)
|
|
|
|
|
|
| > avg_loss:[92m 10.76663 [0m(-0.04476)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93391 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_228.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 114/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:31:23) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.88791 (24.88791)
|
|
|
|
|
|
| > postnet_loss: 14.94401 (14.94401)
|
|
|
|
|
|
| > stopnet_loss: 0.76569 (0.76569)
|
|
|
|
|
|
| > loss: 10.72367 (10.72367)
|
|
|
|
|
|
| > align_error: 0.93394 (0.93394)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.38205 [0m(+0.00081)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.88791 [0m(-0.01609)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 14.94401 [0m(-0.15305)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.76569 [0m(-0.00067)
|
|
|
|
|
|
| > avg_loss:[92m 10.72367 [0m(-0.04296)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93394 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_230.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 115/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:31:38) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.87119 (24.87119)
|
|
|
|
|
|
| > postnet_loss: 14.80095 (14.80095)
|
|
|
|
|
|
| > stopnet_loss: 0.76479 (0.76479)
|
|
|
|
|
|
| > loss: 10.68282 (10.68282)
|
|
|
|
|
|
| > align_error: 0.93396 (0.93396)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.39776 [0m(+0.01571)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.87119 [0m(-0.01672)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 14.80095 [0m(-0.14306)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.76479 [0m(-0.00091)
|
|
|
|
|
|
| > avg_loss:[92m 10.68282 [0m(-0.04085)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93396 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_232.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 116/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:31:53) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.85367 (24.85367)
|
|
|
|
|
|
| > postnet_loss: 14.65728 (14.65728)
|
|
|
|
|
|
| > stopnet_loss: 0.76392 (0.76392)
|
|
|
|
|
|
| > loss: 10.64166 (10.64166)
|
|
|
|
|
|
| > align_error: 0.93399 (0.93399)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.38093 [0m(-0.01683)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.85367 [0m(-0.01752)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 14.65728 [0m(-0.14366)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.76392 [0m(-0.00086)
|
|
|
|
|
|
| > avg_loss:[92m 10.64166 [0m(-0.04116)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93399 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_234.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 117/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:32:09) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.83603 (24.83603)
|
|
|
|
|
|
| > postnet_loss: 14.52243 (14.52243)
|
|
|
|
|
|
| > stopnet_loss: 0.76315 (0.76315)
|
|
|
|
|
|
| > loss: 10.60277 (10.60277)
|
|
|
|
|
|
| > align_error: 0.93401 (0.93401)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.70377 [0m(+0.32284)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.83603 [0m(-0.01763)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 14.52243 [0m(-0.13486)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.76315 [0m(-0.00077)
|
|
|
|
|
|
| > avg_loss:[92m 10.60277 [0m(-0.03890)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93401 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_236.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 118/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:32:38) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.81760 (24.81760)
|
|
|
|
|
|
| > postnet_loss: 14.39176 (14.39176)
|
|
|
|
|
|
| > stopnet_loss: 0.76246 (0.76246)
|
|
|
|
|
|
| > loss: 10.56480 (10.56480)
|
|
|
|
|
|
| > align_error: 0.93404 (0.93404)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.57390 [0m(-0.12987)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.81760 [0m(-0.01844)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 14.39176 [0m(-0.13066)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.76246 [0m(-0.00069)
|
|
|
|
|
|
| > avg_loss:[92m 10.56480 [0m(-0.03797)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93404 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_238.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 119/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:32:54) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.79859 (24.79859)
|
|
|
|
|
|
| > postnet_loss: 14.25896 (14.25896)
|
|
|
|
|
|
| > stopnet_loss: 0.76150 (0.76150)
|
|
|
|
|
|
| > loss: 10.52589 (10.52589)
|
|
|
|
|
|
| > align_error: 0.93407 (0.93407)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.53127 [0m(-0.04263)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.79859 [0m(-0.01901)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 14.25896 [0m(-0.13280)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.76150 [0m(-0.00096)
|
|
|
|
|
|
| > avg_loss:[92m 10.52589 [0m(-0.03891)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93407 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_240.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 120/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:33:10) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.77885 (24.77885)
|
|
|
|
|
|
| > postnet_loss: 14.14054 (14.14054)
|
|
|
|
|
|
| > stopnet_loss: 0.76049 (0.76049)
|
|
|
|
|
|
| > loss: 10.49034 (10.49034)
|
|
|
|
|
|
| > align_error: 0.93410 (0.93410)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.52745 [0m(-0.00382)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.77885 [0m(-0.01974)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 14.14054 [0m(-0.11842)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.76049 [0m(-0.00101)
|
|
|
|
|
|
| > avg_loss:[92m 10.49034 [0m(-0.03555)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93410 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_242.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 121/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:33:26) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.75903 (24.75903)
|
|
|
|
|
|
| > postnet_loss: 14.01721 (14.01721)
|
|
|
|
|
|
| > stopnet_loss: 0.75958 (0.75958)
|
|
|
|
|
|
| > loss: 10.45364 (10.45364)
|
|
|
|
|
|
| > align_error: 0.93413 (0.93413)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.54692 [0m(+0.01947)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.75903 [0m(-0.01982)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 14.01721 [0m(-0.12334)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.75958 [0m(-0.00091)
|
|
|
|
|
|
| > avg_loss:[92m 10.45364 [0m(-0.03670)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93413 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_244.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 122/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:33:41) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.73801 (24.73801)
|
|
|
|
|
|
| > postnet_loss: 13.89961 (13.89961)
|
|
|
|
|
|
| > stopnet_loss: 0.75879 (0.75879)
|
|
|
|
|
|
| > loss: 10.41820 (10.41820)
|
|
|
|
|
|
| > align_error: 0.93415 (0.93415)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.51816 [0m(-0.02876)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.73801 [0m(-0.02101)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 13.89961 [0m(-0.11760)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.75879 [0m(-0.00079)
|
|
|
|
|
|
| > avg_loss:[92m 10.41820 [0m(-0.03544)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93415 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_246.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 123/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:33:56) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.71552 (24.71552)
|
|
|
|
|
|
| > postnet_loss: 13.79109 (13.79109)
|
|
|
|
|
|
| > stopnet_loss: 0.75797 (0.75797)
|
|
|
|
|
|
| > loss: 10.38462 (10.38462)
|
|
|
|
|
|
| > align_error: 0.93418 (0.93418)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.59217 [0m(+0.07401)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.71552 [0m(-0.02249)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 13.79109 [0m(-0.10851)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.75797 [0m(-0.00082)
|
|
|
|
|
|
| > avg_loss:[92m 10.38462 [0m(-0.03358)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93418 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_248.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 124/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:34:11) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.69184 (24.69184)
|
|
|
|
|
|
| > postnet_loss: 13.67974 (13.67974)
|
|
|
|
|
|
| > stopnet_loss: 0.75692 (0.75692)
|
|
|
|
|
|
| > loss: 10.34981 (10.34981)
|
|
|
|
|
|
| > align_error: 0.93421 (0.93421)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.54020 [0m(-0.05197)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.69184 [0m(-0.02368)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 13.67974 [0m(-0.11136)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.75692 [0m(-0.00105)
|
|
|
|
|
|
| > avg_loss:[92m 10.34981 [0m(-0.03481)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93421 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_250.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 125/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:34:28) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0/2 -- GLOBAL_STEP: 250[0m
|
|
|
|
|
|
| > decoder_loss: 28.14487 (28.14487)
|
|
|
|
|
|
| > postnet_loss: 23.95758 (23.95758)
|
|
|
|
|
|
| > stopnet_loss: 0.74521 (0.74521)
|
|
|
|
|
|
| > loss: 13.77082 (13.77082)
|
|
|
|
|
|
| > align_error: 0.95789 (0.95789)
|
|
|
|
|
|
| > grad_norm: 6.26954 (6.26954)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 1.95020 (1.95022)
|
|
|
|
|
|
| > loader_time: 3.93320 (3.93319)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.66791 (24.66791)
|
|
|
|
|
|
| > postnet_loss: 13.57413 (13.57413)
|
|
|
|
|
|
| > stopnet_loss: 0.75562 (0.75562)
|
|
|
|
|
|
| > loss: 10.31613 (10.31613)
|
|
|
|
|
|
| > align_error: 0.93424 (0.93424)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.70751 [0m(+0.16731)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.66791 [0m(-0.02394)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 13.57413 [0m(-0.10561)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.75562 [0m(-0.00129)
|
|
|
|
|
|
| > avg_loss:[92m 10.31613 [0m(-0.03368)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93424 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_252.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 126/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:34:57) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.64308 (24.64308)
|
|
|
|
|
|
| > postnet_loss: 13.46872 (13.46872)
|
|
|
|
|
|
| > stopnet_loss: 0.75464 (0.75464)
|
|
|
|
|
|
| > loss: 10.28259 (10.28259)
|
|
|
|
|
|
| > align_error: 0.93427 (0.93427)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.40032 [0m(-0.30719)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.64308 [0m(-0.02483)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 13.46872 [0m(-0.10541)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.75464 [0m(-0.00099)
|
|
|
|
|
|
| > avg_loss:[92m 10.28259 [0m(-0.03355)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93427 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_254.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 127/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:35:13) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.61713 (24.61713)
|
|
|
|
|
|
| > postnet_loss: 13.37599 (13.37599)
|
|
|
|
|
|
| > stopnet_loss: 0.75351 (0.75351)
|
|
|
|
|
|
| > loss: 10.25179 (10.25179)
|
|
|
|
|
|
| > align_error: 0.93430 (0.93430)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.38094 [0m(-0.01938)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.61713 [0m(-0.02595)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 13.37599 [0m(-0.09273)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.75351 [0m(-0.00113)
|
|
|
|
|
|
| > avg_loss:[92m 10.25179 [0m(-0.03080)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93430 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_256.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 128/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:35:29) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.59013 (24.59013)
|
|
|
|
|
|
| > postnet_loss: 13.29527 (13.29527)
|
|
|
|
|
|
| > stopnet_loss: 0.75232 (0.75232)
|
|
|
|
|
|
| > loss: 10.22367 (10.22367)
|
|
|
|
|
|
| > align_error: 0.93433 (0.93433)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.40416 [0m(+0.02321)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.59013 [0m(-0.02699)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 13.29527 [0m(-0.08072)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.75232 [0m(-0.00119)
|
|
|
|
|
|
| > avg_loss:[92m 10.22367 [0m(-0.02811)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93433 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_258.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 129/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:35:44) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.56199 (24.56199)
|
|
|
|
|
|
| > postnet_loss: 13.20905 (13.20905)
|
|
|
|
|
|
| > stopnet_loss: 0.75101 (0.75101)
|
|
|
|
|
|
| > loss: 10.19377 (10.19377)
|
|
|
|
|
|
| > align_error: 0.93436 (0.93436)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.37682 [0m(-0.02734)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.56199 [0m(-0.02814)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 13.20905 [0m(-0.08622)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.75101 [0m(-0.00132)
|
|
|
|
|
|
| > avg_loss:[92m 10.19377 [0m(-0.02991)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93436 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_260.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 130/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:36:00) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.53213 (24.53213)
|
|
|
|
|
|
| > postnet_loss: 13.12643 (13.12643)
|
|
|
|
|
|
| > stopnet_loss: 0.74978 (0.74978)
|
|
|
|
|
|
| > loss: 10.16441 (10.16441)
|
|
|
|
|
|
| > align_error: 0.93439 (0.93439)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.02092 [0m(+0.64410)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.53213 [0m(-0.02987)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 13.12643 [0m(-0.08262)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.74978 [0m(-0.00123)
|
|
|
|
|
|
| > avg_loss:[92m 10.16441 [0m(-0.02935)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93439 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_262.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 131/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:36:17) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.50153 (24.50153)
|
|
|
|
|
|
| > postnet_loss: 13.04998 (13.04998)
|
|
|
|
|
|
| > stopnet_loss: 0.74841 (0.74841)
|
|
|
|
|
|
| > loss: 10.13629 (10.13629)
|
|
|
|
|
|
| > align_error: 0.93442 (0.93442)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.56871 [0m(-0.45221)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.50153 [0m(-0.03060)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 13.04998 [0m(-0.07645)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.74841 [0m(-0.00137)
|
|
|
|
|
|
| > avg_loss:[92m 10.13629 [0m(-0.02813)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93442 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_264.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 132/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:36:32) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.47037 (24.47037)
|
|
|
|
|
|
| > postnet_loss: 12.97397 (12.97397)
|
|
|
|
|
|
| > stopnet_loss: 0.74710 (0.74710)
|
|
|
|
|
|
| > loss: 10.10819 (10.10819)
|
|
|
|
|
|
| > align_error: 0.93445 (0.93445)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.55167 [0m(-0.01704)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.47037 [0m(-0.03116)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.97397 [0m(-0.07601)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.74710 [0m(-0.00131)
|
|
|
|
|
|
| > avg_loss:[92m 10.10819 [0m(-0.02810)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93445 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_266.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 133/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:36:47) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.43732 (24.43732)
|
|
|
|
|
|
| > postnet_loss: 12.90439 (12.90439)
|
|
|
|
|
|
| > stopnet_loss: 0.74572 (0.74572)
|
|
|
|
|
|
| > loss: 10.08115 (10.08115)
|
|
|
|
|
|
| > align_error: 0.93449 (0.93449)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.35288 [0m(+0.80121)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.43732 [0m(-0.03306)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.90439 [0m(-0.06958)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.74572 [0m(-0.00138)
|
|
|
|
|
|
| > avg_loss:[92m 10.08115 [0m(-0.02704)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93449 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_268.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 134/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:37:16) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.40205 (24.40205)
|
|
|
|
|
|
| > postnet_loss: 12.84301 (12.84301)
|
|
|
|
|
|
| > stopnet_loss: 0.74424 (0.74424)
|
|
|
|
|
|
| > loss: 10.05551 (10.05551)
|
|
|
|
|
|
| > align_error: 0.93452 (0.93452)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.37588 [0m(-0.97701)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.40205 [0m(-0.03526)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.84301 [0m(-0.06138)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.74424 [0m(-0.00148)
|
|
|
|
|
|
| > avg_loss:[92m 10.05551 [0m(-0.02564)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93452 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_270.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 135/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:37:31) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.36577 (24.36577)
|
|
|
|
|
|
| > postnet_loss: 12.78301 (12.78301)
|
|
|
|
|
|
| > stopnet_loss: 0.74244 (0.74244)
|
|
|
|
|
|
| > loss: 10.02964 (10.02964)
|
|
|
|
|
|
| > align_error: 0.93455 (0.93455)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.79215 [0m(+0.41627)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.36577 [0m(-0.03628)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.78301 [0m(-0.05999)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.74244 [0m(-0.00180)
|
|
|
|
|
|
| > avg_loss:[92m 10.02964 [0m(-0.02587)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93455 [0m(+0.00003)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_272.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 136/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:37:47) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.32769 (24.32769)
|
|
|
|
|
|
| > postnet_loss: 12.72485 (12.72485)
|
|
|
|
|
|
| > stopnet_loss: 0.74075 (0.74075)
|
|
|
|
|
|
| > loss: 10.00389 (10.00389)
|
|
|
|
|
|
| > align_error: 0.93459 (0.93459)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.38061 [0m(-0.41154)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.32769 [0m(-0.03808)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.72485 [0m(-0.05816)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.74075 [0m(-0.00169)
|
|
|
|
|
|
| > avg_loss:[92m 10.00389 [0m(-0.02575)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93459 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_274.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 137/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:38:02) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 1/2 -- GLOBAL_STEP: 275[0m
|
|
|
|
|
|
| > decoder_loss: 29.78144 (29.78144)
|
|
|
|
|
|
| > postnet_loss: 22.82139 (22.82139)
|
|
|
|
|
|
| > stopnet_loss: 0.73713 (0.73713)
|
|
|
|
|
|
| > loss: 13.88784 (13.88784)
|
|
|
|
|
|
| > align_error: 0.97020 (0.97020)
|
|
|
|
|
|
| > grad_norm: 5.63345 (5.63345)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 2.32140 (2.32137)
|
|
|
|
|
|
| > loader_time: 0.00550 (0.00552)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.28755 (24.28755)
|
|
|
|
|
|
| > postnet_loss: 12.66898 (12.66898)
|
|
|
|
|
|
| > stopnet_loss: 0.73894 (0.73894)
|
|
|
|
|
|
| > loss: 9.97807 (9.97807)
|
|
|
|
|
|
| > align_error: 0.93462 (0.93462)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.38795 [0m(+0.00734)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.28755 [0m(-0.04014)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.66898 [0m(-0.05587)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.73894 [0m(-0.00182)
|
|
|
|
|
|
| > avg_loss:[92m 9.97807 [0m(-0.02582)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93462 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_276.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 138/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:38:17) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.24691 (24.24691)
|
|
|
|
|
|
| > postnet_loss: 12.61333 (12.61333)
|
|
|
|
|
|
| > stopnet_loss: 0.73689 (0.73689)
|
|
|
|
|
|
| > loss: 9.95195 (9.95195)
|
|
|
|
|
|
| > align_error: 0.93466 (0.93466)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.38630 [0m(-0.00165)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.24691 [0m(-0.04064)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.61333 [0m(-0.05565)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.73689 [0m(-0.00204)
|
|
|
|
|
|
| > avg_loss:[92m 9.95195 [0m(-0.02612)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93466 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_278.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 139/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:38:32) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.20509 (24.20509)
|
|
|
|
|
|
| > postnet_loss: 12.57103 (12.57103)
|
|
|
|
|
|
| > stopnet_loss: 0.73505 (0.73505)
|
|
|
|
|
|
| > loss: 9.92908 (9.92908)
|
|
|
|
|
|
| > align_error: 0.93470 (0.93470)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 2.12122 [0m(+1.73492)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.20509 [0m(-0.04181)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.57103 [0m(-0.04230)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.73505 [0m(-0.00184)
|
|
|
|
|
|
| > avg_loss:[92m 9.92908 [0m(-0.02287)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93470 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_280.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 140/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:40:05) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.16056 (24.16056)
|
|
|
|
|
|
| > postnet_loss: 12.52215 (12.52215)
|
|
|
|
|
|
| > stopnet_loss: 0.73308 (0.73308)
|
|
|
|
|
|
| > loss: 9.90376 (9.90376)
|
|
|
|
|
|
| > align_error: 0.93473 (0.93473)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.53318 [0m(-1.58804)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.16056 [0m(-0.04454)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.52215 [0m(-0.04887)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.73308 [0m(-0.00197)
|
|
|
|
|
|
| > avg_loss:[92m 9.90376 [0m(-0.02532)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93473 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_282.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 141/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:40:21) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.11544 (24.11544)
|
|
|
|
|
|
| > postnet_loss: 12.47741 (12.47741)
|
|
|
|
|
|
| > stopnet_loss: 0.73080 (0.73080)
|
|
|
|
|
|
| > loss: 9.87901 (9.87901)
|
|
|
|
|
|
| > align_error: 0.93477 (0.93477)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.54237 [0m(+0.00919)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.11544 [0m(-0.04511)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.47741 [0m(-0.04475)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.73080 [0m(-0.00229)
|
|
|
|
|
|
| > avg_loss:[92m 9.87901 [0m(-0.02475)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93477 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_284.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 142/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:40:36) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.06737 (24.06737)
|
|
|
|
|
|
| > postnet_loss: 12.43430 (12.43430)
|
|
|
|
|
|
| > stopnet_loss: 0.72849 (0.72849)
|
|
|
|
|
|
| > loss: 9.85390 (9.85390)
|
|
|
|
|
|
| > align_error: 0.93481 (0.93481)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.54418 [0m(+0.00181)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.06737 [0m(-0.04808)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.43430 [0m(-0.04311)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.72849 [0m(-0.00231)
|
|
|
|
|
|
| > avg_loss:[92m 9.85390 [0m(-0.02511)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93481 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_286.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 143/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:40:52) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 24.01711 (24.01711)
|
|
|
|
|
|
| > postnet_loss: 12.39214 (12.39214)
|
|
|
|
|
|
| > stopnet_loss: 0.72584 (0.72584)
|
|
|
|
|
|
| > loss: 9.82815 (9.82815)
|
|
|
|
|
|
| > align_error: 0.93485 (0.93485)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.54488 [0m(+0.00070)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 24.01711 [0m(-0.05025)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.39214 [0m(-0.04216)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.72584 [0m(-0.00265)
|
|
|
|
|
|
| > avg_loss:[92m 9.82815 [0m(-0.02575)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93485 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_288.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 144/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:41:07) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.96517 (23.96517)
|
|
|
|
|
|
| > postnet_loss: 12.35137 (12.35137)
|
|
|
|
|
|
| > stopnet_loss: 0.72327 (0.72327)
|
|
|
|
|
|
| > loss: 9.80241 (9.80241)
|
|
|
|
|
|
| > align_error: 0.93489 (0.93489)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.47806 [0m(+0.93318)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.96517 [0m(-0.05194)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.35137 [0m(-0.04077)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.72327 [0m(-0.00257)
|
|
|
|
|
|
| > avg_loss:[92m 9.80241 [0m(-0.02574)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93489 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_290.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 145/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:42:35) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.91213 (23.91213)
|
|
|
|
|
|
| > postnet_loss: 12.32036 (12.32036)
|
|
|
|
|
|
| > stopnet_loss: 0.72093 (0.72093)
|
|
|
|
|
|
| > loss: 9.77905 (9.77905)
|
|
|
|
|
|
| > align_error: 0.93493 (0.93493)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.64957 [0m(+0.17150)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.91213 [0m(-0.05304)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.32036 [0m(-0.03101)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.72093 [0m(-0.00234)
|
|
|
|
|
|
| > avg_loss:[92m 9.77905 [0m(-0.02336)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93493 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_292.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 146/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:44:04) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.85706 (23.85706)
|
|
|
|
|
|
| > postnet_loss: 12.28907 (12.28907)
|
|
|
|
|
|
| > stopnet_loss: 0.71845 (0.71845)
|
|
|
|
|
|
| > loss: 9.75498 (9.75498)
|
|
|
|
|
|
| > align_error: 0.93497 (0.93497)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.65575 [0m(+0.00618)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.85706 [0m(-0.05508)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.28907 [0m(-0.03129)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.71845 [0m(-0.00248)
|
|
|
|
|
|
| > avg_loss:[92m 9.75498 [0m(-0.02408)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93497 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_294.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 147/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:45:30) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.79927 (23.79927)
|
|
|
|
|
|
| > postnet_loss: 12.25081 (12.25081)
|
|
|
|
|
|
| > stopnet_loss: 0.71556 (0.71556)
|
|
|
|
|
|
| > loss: 9.72808 (9.72808)
|
|
|
|
|
|
| > align_error: 0.93501 (0.93501)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.40748 [0m(-1.24827)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.79927 [0m(-0.05779)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.25081 [0m(-0.03826)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.71556 [0m(-0.00288)
|
|
|
|
|
|
| > avg_loss:[92m 9.72808 [0m(-0.02690)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93501 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_296.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 148/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:45:47) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.73929 (23.73929)
|
|
|
|
|
|
| > postnet_loss: 12.21506 (12.21506)
|
|
|
|
|
|
| > stopnet_loss: 0.71260 (0.71260)
|
|
|
|
|
|
| > loss: 9.70119 (9.70119)
|
|
|
|
|
|
| > align_error: 0.93506 (0.93506)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.37874 [0m(-0.02875)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.73929 [0m(-0.05998)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.21506 [0m(-0.03575)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.71260 [0m(-0.00296)
|
|
|
|
|
|
| > avg_loss:[92m 9.70119 [0m(-0.02689)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93506 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_298.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 149/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:46:08) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.67859 (23.67859)
|
|
|
|
|
|
| > postnet_loss: 12.18098 (12.18098)
|
|
|
|
|
|
| > stopnet_loss: 0.70969 (0.70969)
|
|
|
|
|
|
| > loss: 9.67458 (9.67458)
|
|
|
|
|
|
| > align_error: 0.93510 (0.93510)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 0.38760 [0m(+0.00886)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.67859 [0m(-0.06070)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.18098 [0m(-0.03408)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.70969 [0m(-0.00291)
|
|
|
|
|
|
| > avg_loss:[92m 9.67458 [0m(-0.02660)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93510 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_300.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 150/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:46:24) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0/2 -- GLOBAL_STEP: 300[0m
|
|
|
|
|
|
| > decoder_loss: 27.24969 (27.24969)
|
|
|
|
|
|
| > postnet_loss: 21.39986 (21.39986)
|
|
|
|
|
|
| > stopnet_loss: 0.70556 (0.70556)
|
|
|
|
|
|
| > loss: 12.86795 (12.86795)
|
|
|
|
|
|
| > align_error: 0.95804 (0.95804)
|
|
|
|
|
|
| > grad_norm: 4.39074 (4.39074)
|
|
|
|
|
|
| > current_lr: 0.00000
|
|
|
|
|
|
| > step_time: 0.75780 (0.75777)
|
|
|
|
|
|
| > loader_time: 71.91820 (71.91817)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.61572 (23.61572)
|
|
|
|
|
|
| > postnet_loss: 12.14773 (12.14773)
|
|
|
|
|
|
| > stopnet_loss: 0.70669 (0.70669)
|
|
|
|
|
|
| > loss: 9.64756 (9.64756)
|
|
|
|
|
|
| > align_error: 0.93514 (0.93514)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.47811 [0m(+1.09051)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.61572 [0m(-0.06286)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.14773 [0m(-0.03325)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.70669 [0m(-0.00300)
|
|
|
|
|
|
| > avg_loss:[92m 9.64756 [0m(-0.02703)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93514 [0m(+0.00004)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_302.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 151/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:47:52) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.54928 (23.54928)
|
|
|
|
|
|
| > postnet_loss: 12.11641 (12.11641)
|
|
|
|
|
|
| > stopnet_loss: 0.70334 (0.70334)
|
|
|
|
|
|
| > loss: 9.61977 (9.61977)
|
|
|
|
|
|
| > align_error: 0.93519 (0.93519)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.52243 [0m(+0.04432)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.54928 [0m(-0.06644)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.11641 [0m(-0.03132)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.70334 [0m(-0.00335)
|
|
|
|
|
|
| > avg_loss:[92m 9.61977 [0m(-0.02779)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93519 [0m(+0.00005)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_304.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 152/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:49:19) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.48001 (23.48001)
|
|
|
|
|
|
| > postnet_loss: 12.08576 (12.08576)
|
|
|
|
|
|
| > stopnet_loss: 0.69999 (0.69999)
|
|
|
|
|
|
| > loss: 9.59144 (9.59144)
|
|
|
|
|
|
| > align_error: 0.93524 (0.93524)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.62266 [0m(+0.10022)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.48001 [0m(-0.06927)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.08576 [0m(-0.03065)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.69999 [0m(-0.00335)
|
|
|
|
|
|
| > avg_loss:[92m 9.59144 [0m(-0.02833)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93524 [0m(+0.00005)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_306.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 153/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:50:44) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.40978 (23.40978)
|
|
|
|
|
|
| > postnet_loss: 12.05594 (12.05594)
|
|
|
|
|
|
| > stopnet_loss: 0.69660 (0.69660)
|
|
|
|
|
|
| > loss: 9.56303 (9.56303)
|
|
|
|
|
|
| > align_error: 0.93528 (0.93528)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 1.45256 [0m(-0.17010)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.40978 [0m(-0.07023)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.05594 [0m(-0.02982)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.69660 [0m(-0.00339)
|
|
|
|
|
|
| > avg_loss:[92m 9.56303 [0m(-0.02841)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93528 [0m(+0.00005)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_308.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 154/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:52:11) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.33634 (23.33634)
|
|
|
|
|
|
| > postnet_loss: 12.01772 (12.01772)
|
|
|
|
|
|
| > stopnet_loss: 0.69303 (0.69303)
|
|
|
|
|
|
| > loss: 9.53155 (9.53155)
|
|
|
|
|
|
| > align_error: 0.93533 (0.93533)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.45948 [0m(+0.00693)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.33634 [0m(-0.07344)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 12.01772 [0m(-0.03822)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.69303 [0m(-0.00357)
|
|
|
|
|
|
| > avg_loss:[92m 9.53155 [0m(-0.03148)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93533 [0m(+0.00005)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_310.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 155/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:53:38) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.26028 (23.26028)
|
|
|
|
|
|
| > postnet_loss: 11.98090 (11.98090)
|
|
|
|
|
|
| > stopnet_loss: 0.68924 (0.68924)
|
|
|
|
|
|
| > loss: 9.49953 (9.49953)
|
|
|
|
|
|
| > align_error: 0.93538 (0.93538)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[92m 0.40167 [0m(-1.05781)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.26028 [0m(-0.07607)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 11.98090 [0m(-0.03682)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.68924 [0m(-0.00380)
|
|
|
|
|
|
| > avg_loss:[92m 9.49953 [0m(-0.03202)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93538 [0m(+0.00005)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_312.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 156/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:53:54) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.18120 (23.18120)
|
|
|
|
|
|
| > postnet_loss: 11.94405 (11.94405)
|
|
|
|
|
|
| > stopnet_loss: 0.68525 (0.68525)
|
|
|
|
|
|
| > loss: 9.46656 (9.46656)
|
|
|
|
|
|
| > align_error: 0.93543 (0.93543)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 1.50391 [0m(+1.10224)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.18120 [0m(-0.07908)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 11.94405 [0m(-0.03685)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.68525 [0m(-0.00399)
|
|
|
|
|
|
| > avg_loss:[92m 9.46656 [0m(-0.03297)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93543 [0m(+0.00005)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_314.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 157/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:55:21) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.09868 (23.09868)
|
|
|
|
|
|
| > postnet_loss: 11.90392 (11.90392)
|
|
|
|
|
|
| > stopnet_loss: 0.68117 (0.68117)
|
|
|
|
|
|
| > loss: 9.43182 (9.43182)
|
|
|
|
|
|
| > align_error: 0.93549 (0.93549)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m
|
|
|
|
|
|
| > avg_loader_time:[91m 2.29519 [0m(+0.79127)
|
|
|
|
|
|
| > avg_decoder_loss:[92m 23.09868 [0m(-0.08252)
|
|
|
|
|
|
| > avg_postnet_loss:[92m 11.90392 [0m(-0.04013)
|
|
|
|
|
|
| > avg_stopnet_loss:[92m 0.68117 [0m(-0.00408)
|
|
|
|
|
|
| > avg_loss:[92m 9.43182 [0m(-0.03474)
|
|
|
|
|
|
| > avg_align_error:[91m 0.93549 [0m(+0.00005)
|
|
|
|
|
|
|
|
|
|
|
|
> BEST MODEL : output/run-May-12-2023_06+01AM-0429ab9/best_model_316.pth
|
|
|
|
|
|
|
|
|
|
|
|
[4m[1m > EPOCH: 158/1000[0m
|
|
|
|
|
|
--> output/run-May-12-2023_06+01AM-0429ab9
|
|
|
|
|
|
|
|
|
|
|
|
[1m > TRAINING (2023-05-12 06:56:48) [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m > EVALUATION [0m
|
|
|
|
|
|
|
|
|
|
|
|
[1m --> STEP: 0[0m
|
|
|
|
|
|
| > decoder_loss: 23.01368 (23.01368)
|
|
|
|
|
|
| > postnet_loss: 11.86693 (11.86693)
|
|
|
|
|
|
| > stopnet_loss: 0.67710 (0.67710)
|
|
|
|
|
|
| > loss: 9.39726 (9.39726)
|
|
|
|
|
|
| > align_error: 0.93554 (0.93554)
|
|
|
|
|
|
|