Upload 8 files

checkpoints/log_LRS2_lip_tfgridnet_3spk/config.yaml

@@ -0,0 +1,50 @@
+## Config file
+
+# Log 
+seed: 777
+use_cuda: 1           # 1 for True, 0 for False
+
+# dataset
+speaker_no: 3
+mix_lst_path: ./data/LRS2/mixture_data_list_3mix.csv
+audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/LRS2/audio_clean/
+reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/LRS2/mvlrs_v1/
+audio_sr: 16000
+ref_sr: 25
+
+# dataloader
+num_workers: 4
+batch_size: 1         # 8-GPU training with a total effective batch size of 8
+accu_grad: 0
+effec_batch_size: 4   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
+max_length: 3         # truncate the utterances in dataloader, in seconds 
+
+# network settings
+init_from: None       # 'None' or a log name 'log_2024-07-22(18:12:13)'
+causal: 0             # 1 for True, 0 for False
+network_reference:
+  cue: lip            # lip or speech or gesture or EEG
+  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
+  emb_size: 256       # resnet18:256
+network_audio:
+  backbone: av_tfgridnet
+  n_fft: 256
+  stride: 128
+  window: "hann"
+  use_builtin_complex: False
+  n_srcs:  1
+  n_imics: 1
+  n_layers: 6
+  lstm_hidden_units: 192
+  attn_n_head: 4
+  attn_qk_output_channel: 4
+  emb_dim: 48
+  emb_ks: 4
+  emb_hs: 1
+  activation: "prelu"
+
+# optimizer
+loss_type: sisdr      # "snr", "sisdr", "hybrid"
+init_learning_rate: 0.001
+max_epoch: 150
+clip_grad_norm: 5

checkpoints/log_LRS2_lip_tfgridnet_3spk/last_best_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:623f54eb793683defc579c2ff0d9ba9fa26100e016fe82278d274d57a57fc5bd
+size 160433102

checkpoints/log_LRS2_lip_tfgridnet_3spk/last_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:360ac77f05d11f21539dda902fd2d8a7857ece03e6bd4946c818a436d776b45b
+size 160425982

checkpoints/log_LRS2_lip_tfgridnet_3spk/log_2024-10-01(15:53:50).txt

@@ -0,0 +1,503 @@
+## Config file
+
+# Log 
+seed: 777
+use_cuda: 1           # 1 for True, 0 for False
+
+# dataset
+speaker_no: 3
+mix_lst_path: ./data/LRS2/mixture_data_list_3mix.csv
+audio_direc: /mnt/nas_sg/mit_sg/zexu.pan/datasets/LRS2/audio_clean/
+reference_direc: /mnt/nas_sg/mit_sg/zexu.pan/datasets/LRS2/mvlrs_v1/
+audio_sr: 16000
+ref_sr: 25
+
+# dataloader
+num_workers: 4
+batch_size: 1         # 8-GPU training with a total effective batch size of 8
+accu_grad: 0
+effec_batch_size: 4   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
+max_length: 3         # truncate the utterances in dataloader, in seconds 
+
+# network settings
+init_from: None       # 'None' or a log name 'log_2024-07-22(18:12:13)'
+causal: 0             # 1 for True, 0 for False
+network_reference:
+  cue: lip            # lip or speech or gesture or EEG
+  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
+  emb_size: 256       # resnet18:256
+network_audio:
+  backbone: tfgridnet
+  n_fft: 256
+  stride: 128
+  window: "hann"
+  use_builtin_complex: False
+  n_srcs:  1
+  n_imics: 1
+  n_layers: 6
+  lstm_hidden_units: 192
+  attn_n_head: 4
+  attn_qk_output_channel: 4
+  emb_dim: 48
+  emb_ks: 4
+  emb_hs: 1
+  activation: "prelu"
+
+# optimizer
+loss_type: sisdr      # "snr", "sisdr", "hybrid"
+init_learning_rate: 0.001
+max_epoch: 150
+clip_grad_norm: 5
+W1001 15:53:53.420768 139891169711936 torch/distributed/run.py:757] 
+W1001 15:53:53.420768 139891169711936 torch/distributed/run.py:757] *****************************************
+W1001 15:53:53.420768 139891169711936 torch/distributed/run.py:757] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W1001 15:53:53.420768 139891169711936 torch/distributed/run.py:757] *****************************************
+started on checkpoints/log_2024-10-01(15:53:50)
+
+namespace(seed=777, use_cuda=1, config=[<yamlargparse.Path object at 0x7f20217b4e80>], checkpoint_dir='checkpoints/log_2024-10-01(15:53:50)', train_from_last_checkpoint=0, loss_type='sisdr', init_learning_rate=0.001, lr_warmup=0, max_epoch=150, clip_grad_norm=5.0, batch_size=1, accu_grad=0, effec_batch_size=4, max_length=3, num_workers=4, causal=0, network_reference=namespace(cue='lip', backbone='resnet18', emb_size=256), network_audio=namespace(backbone='tfgridnet', n_fft=256, stride=128, window='hann', use_builtin_complex=False, n_srcs=1, n_imics=1, n_layers=6, lstm_hidden_units=192, attn_n_head=4, attn_qk_output_channel=4, emb_dim=48, emb_ks=4, emb_hs=1, activation='prelu'), init_from='None', mix_lst_path='./data/LRS2/mixture_data_list_3mix.csv', audio_direc='/mnt/nas_sg/mit_sg/zexu.pan/datasets/LRS2/audio_clean/', reference_direc='/mnt/nas_sg/mit_sg/zexu.pan/datasets/LRS2/mvlrs_v1/', speaker_no=3, audio_sr=16000, ref_sr=25, local_rank=0, distributed=True, world_size=8, device=device(type='cuda'))
+network_wrapper(
+  (sep_network): TFGridNetV3(
+    (enc): STFTEncoder(
+      (stft): Stft(n_fft=256, win_length=256, hop_length=128, center=True, normalized=False, onesided=True)
+    )
+    (dec): STFTDecoder(
+      (stft): Stft(n_fft=256, win_length=256, hop_length=128, center=True, normalized=False, onesided=True)
+    )
+    (conv): Sequential(
+      (0): Conv2d(2, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+      (1): GroupNorm(1, 48, eps=1e-05, affine=True)
+    )
+    (blocks): ModuleList(
+      (0-5): 6 x GridNetV3Block(
+        (intra_norm): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
+        (intra_rnn): LSTM(192, 192, batch_first=True, bidirectional=True)
+        (intra_linear): ConvTranspose1d(384, 48, kernel_size=(4,), stride=(1,))
+        (inter_norm): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
+        (inter_rnn): LSTM(192, 192, batch_first=True, bidirectional=True)
+        (inter_linear): ConvTranspose1d(384, 48, kernel_size=(4,), stride=(1,))
+        (attn_conv_Q): Conv2d(48, 16, kernel_size=(1, 1), stride=(1, 1))
+        (attn_norm_Q): AllHeadPReLULayerNormalization4DC(
+          (act): PReLU(num_parameters=4)
+        )
+        (attn_conv_K): Conv2d(48, 16, kernel_size=(1, 1), stride=(1, 1))
+        (attn_norm_K): AllHeadPReLULayerNormalization4DC(
+          (act): PReLU(num_parameters=4)
+        )
+        (attn_conv_V): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))
+        (attn_norm_V): AllHeadPReLULayerNormalization4DC(
+          (act): PReLU(num_parameters=4)
+        )
+        (attn_concat_proj): Sequential(
+          (0): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))
+          (1): PReLU(num_parameters=1)
+          (2): LayerNormalization()
+        )
+      )
+    )
+    (deconv): ConvTranspose2d(48, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (av_conv): ModuleList(
+      (0-5): 6 x Linear(in_features=304, out_features=48, bias=True)
+    )
+  )
+  (ref_encoder): Visual_encoder(
+    (v_frontend): VisualFrontend(
+      (frontend3D): Sequential(
+        (0): Conv3d(1, 64, kernel_size=(5, 7, 7), stride=(1, 2, 2), padding=(2, 3, 3), bias=False)
+        (1): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        (2): ReLU()
+        (3): MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1), dilation=1, ceil_mode=False)
+      )
+      (resnet): ResNet(
+        (layer1): ResNetLayer(
+          (conv1a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
+          (outbna): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer2): ResNetLayer(
+          (conv1a): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer3): ResNetLayer(
+          (conv1a): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer4): ResNetLayer(
+          (conv1a): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (avgpool): AvgPool2d(kernel_size=(4, 4), stride=(1, 1), padding=0)
+      )
+    )
+    (v_ds): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+    (visual_conv): Sequential(
+      (0): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (1): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (2): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (3): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (4): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+    )
+  )
+)
+
+Total number of parameters: 20780101 
+
+
+Total number of trainable parameters: 9595013 
+
+Start new training from scratch
+[rank1]:[W reducer.cpp:1389] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank3]:[W reducer.cpp:1389] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank5]:[W reducer.cpp:1389] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank2]:[W reducer.cpp:1389] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank6]:[W reducer.cpp:1389] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank4]:[W reducer.cpp:1389] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank7]:[W reducer.cpp:1389] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank0]:[W reducer.cpp:1389] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+Train Summary | End of Epoch 1 | Time 10648.35s | Train Loss 0.798
+Valid Summary | End of Epoch 1 | Time 611.94s | Valid Loss -0.670
+Test Summary | End of Epoch 1 | Time 303.72s | Test Loss -0.847
+Fund new best model, dict saved
+Train Summary | End of Epoch 2 | Time 10657.50s | Train Loss -1.751
+Valid Summary | End of Epoch 2 | Time 612.09s | Valid Loss -2.574
+Test Summary | End of Epoch 2 | Time 304.01s | Test Loss -2.965
+Fund new best model, dict saved
+Train Summary | End of Epoch 3 | Time 10659.67s | Train Loss -3.420
+Valid Summary | End of Epoch 3 | Time 612.06s | Valid Loss -4.035
+Test Summary | End of Epoch 3 | Time 303.92s | Test Loss -4.392
+Fund new best model, dict saved
+Train Summary | End of Epoch 4 | Time 10657.45s | Train Loss -4.505
+Valid Summary | End of Epoch 4 | Time 612.09s | Valid Loss -4.946
+Test Summary | End of Epoch 4 | Time 304.13s | Test Loss -5.418
+Fund new best model, dict saved
+Train Summary | End of Epoch 5 | Time 10660.00s | Train Loss -5.172
+Valid Summary | End of Epoch 5 | Time 612.05s | Valid Loss -5.432
+Test Summary | End of Epoch 5 | Time 304.03s | Test Loss -5.920
+Fund new best model, dict saved
+Train Summary | End of Epoch 6 | Time 10658.66s | Train Loss -5.674
+Valid Summary | End of Epoch 6 | Time 612.21s | Valid Loss -5.829
+Test Summary | End of Epoch 6 | Time 303.88s | Test Loss -6.409
+Fund new best model, dict saved
+Train Summary | End of Epoch 7 | Time 10658.47s | Train Loss -6.028
+Valid Summary | End of Epoch 7 | Time 611.67s | Valid Loss -6.065
+Test Summary | End of Epoch 7 | Time 303.84s | Test Loss -6.727
+Fund new best model, dict saved
+Train Summary | End of Epoch 8 | Time 10658.61s | Train Loss -6.365
+Valid Summary | End of Epoch 8 | Time 612.25s | Valid Loss -6.345
+Test Summary | End of Epoch 8 | Time 303.94s | Test Loss -7.031
+Fund new best model, dict saved
+Train Summary | End of Epoch 9 | Time 10656.89s | Train Loss -6.661
+Valid Summary | End of Epoch 9 | Time 611.95s | Valid Loss -6.569
+Test Summary | End of Epoch 9 | Time 304.14s | Test Loss -7.234
+Fund new best model, dict saved
+Train Summary | End of Epoch 10 | Time 10657.06s | Train Loss -6.882
+Valid Summary | End of Epoch 10 | Time 612.12s | Valid Loss -6.834
+Test Summary | End of Epoch 10 | Time 304.11s | Test Loss -7.553
+Fund new best model, dict saved
+Train Summary | End of Epoch 11 | Time 10655.25s | Train Loss -7.082
+Valid Summary | End of Epoch 11 | Time 611.78s | Valid Loss -7.065
+Test Summary | End of Epoch 11 | Time 304.19s | Test Loss -7.669
+Fund new best model, dict saved
+Train Summary | End of Epoch 12 | Time 10657.47s | Train Loss -7.274
+Valid Summary | End of Epoch 12 | Time 612.17s | Valid Loss -7.168
+Test Summary | End of Epoch 12 | Time 304.15s | Test Loss -7.859
+Fund new best model, dict saved
+Train Summary | End of Epoch 13 | Time 10657.14s | Train Loss -7.426
+Valid Summary | End of Epoch 13 | Time 611.98s | Valid Loss -7.124
+Test Summary | End of Epoch 13 | Time 304.07s | Test Loss -7.863
+Train Summary | End of Epoch 14 | Time 10657.35s | Train Loss -7.545
+Valid Summary | End of Epoch 14 | Time 612.10s | Valid Loss -7.389
+Test Summary | End of Epoch 14 | Time 303.98s | Test Loss -8.066
+Fund new best model, dict saved
+Train Summary | End of Epoch 15 | Time 10655.98s | Train Loss -7.667
+Valid Summary | End of Epoch 15 | Time 611.84s | Valid Loss -7.454
+Test Summary | End of Epoch 15 | Time 303.98s | Test Loss -8.181
+Fund new best model, dict saved
+Train Summary | End of Epoch 16 | Time 10656.82s | Train Loss -7.756
+Valid Summary | End of Epoch 16 | Time 611.77s | Valid Loss -7.408
+Test Summary | End of Epoch 16 | Time 304.19s | Test Loss -8.045
+Train Summary | End of Epoch 17 | Time 10657.17s | Train Loss -7.853
+Valid Summary | End of Epoch 17 | Time 611.99s | Valid Loss -7.559
+Test Summary | End of Epoch 17 | Time 304.10s | Test Loss -8.310
+Fund new best model, dict saved
+Train Summary | End of Epoch 18 | Time 10656.45s | Train Loss -7.899
+Valid Summary | End of Epoch 18 | Time 611.96s | Valid Loss -7.622
+Test Summary | End of Epoch 18 | Time 304.08s | Test Loss -8.364
+Fund new best model, dict saved
+Train Summary | End of Epoch 19 | Time 10657.14s | Train Loss -7.992
+Valid Summary | End of Epoch 19 | Time 612.15s | Valid Loss -7.821
+Test Summary | End of Epoch 19 | Time 304.16s | Test Loss -8.521
+Fund new best model, dict saved
+Train Summary | End of Epoch 20 | Time 10656.52s | Train Loss -8.046
+Valid Summary | End of Epoch 20 | Time 611.83s | Valid Loss -7.770
+Test Summary | End of Epoch 20 | Time 303.92s | Test Loss -8.574
+Train Summary | End of Epoch 21 | Time 10655.12s | Train Loss -7.968
+Valid Summary | End of Epoch 21 | Time 611.92s | Valid Loss -7.635
+Test Summary | End of Epoch 21 | Time 304.04s | Test Loss -8.250
+Train Summary | End of Epoch 22 | Time 10657.13s | Train Loss -8.084
+Valid Summary | End of Epoch 22 | Time 611.90s | Valid Loss -7.788
+Test Summary | End of Epoch 22 | Time 304.11s | Test Loss -8.518
+Train Summary | End of Epoch 23 | Time 10655.92s | Train Loss -8.129
+Valid Summary | End of Epoch 23 | Time 612.01s | Valid Loss -7.663
+Test Summary | End of Epoch 23 | Time 304.23s | Test Loss -8.161
+Train Summary | End of Epoch 24 | Time 10657.57s | Train Loss -8.117
+Valid Summary | End of Epoch 24 | Time 611.89s | Valid Loss -7.801
+Test Summary | End of Epoch 24 | Time 304.07s | Test Loss -8.504
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000500
+Train Summary | End of Epoch 25 | Time 10662.95s | Train Loss -8.577
+Valid Summary | End of Epoch 25 | Time 611.80s | Valid Loss -8.209
+Test Summary | End of Epoch 25 | Time 304.21s | Test Loss -9.011
+Fund new best model, dict saved
+Train Summary | End of Epoch 26 | Time 10665.89s | Train Loss -8.824
+Valid Summary | End of Epoch 26 | Time 611.63s | Valid Loss -8.406
+Test Summary | End of Epoch 26 | Time 304.24s | Test Loss -9.215
+Fund new best model, dict saved
+Train Summary | End of Epoch 27 | Time 10663.44s | Train Loss -8.947
+Valid Summary | End of Epoch 27 | Time 611.91s | Valid Loss -8.379
+Test Summary | End of Epoch 27 | Time 304.43s | Test Loss -9.244
+Train Summary | End of Epoch 28 | Time 10663.82s | Train Loss -9.064
+Valid Summary | End of Epoch 28 | Time 611.97s | Valid Loss -8.507
+Test Summary | End of Epoch 28 | Time 304.21s | Test Loss -9.352
+Fund new best model, dict saved
+Train Summary | End of Epoch 29 | Time 10662.40s | Train Loss -9.153
+Valid Summary | End of Epoch 29 | Time 611.80s | Valid Loss -8.476
+Test Summary | End of Epoch 29 | Time 304.05s | Test Loss -9.340
+Train Summary | End of Epoch 30 | Time 10663.15s | Train Loss -9.249
+Valid Summary | End of Epoch 30 | Time 612.03s | Valid Loss -8.474
+Test Summary | End of Epoch 30 | Time 304.14s | Test Loss -9.398
+Train Summary | End of Epoch 31 | Time 10662.70s | Train Loss -9.324
+Valid Summary | End of Epoch 31 | Time 611.83s | Valid Loss -8.623
+Test Summary | End of Epoch 31 | Time 304.21s | Test Loss -9.531
+Fund new best model, dict saved
+Train Summary | End of Epoch 32 | Time 10665.05s | Train Loss -9.391
+Valid Summary | End of Epoch 32 | Time 611.73s | Valid Loss -8.651
+Test Summary | End of Epoch 32 | Time 303.92s | Test Loss -9.543
+Fund new best model, dict saved
+Train Summary | End of Epoch 33 | Time 10662.53s | Train Loss -9.476
+Valid Summary | End of Epoch 33 | Time 611.92s | Valid Loss -8.705
+Test Summary | End of Epoch 33 | Time 304.13s | Test Loss -9.607
+Fund new best model, dict saved
+Train Summary | End of Epoch 34 | Time 10661.27s | Train Loss -9.530
+Valid Summary | End of Epoch 34 | Time 611.97s | Valid Loss -8.828
+Test Summary | End of Epoch 34 | Time 304.28s | Test Loss -9.661
+Fund new best model, dict saved
+Train Summary | End of Epoch 35 | Time 10661.89s | Train Loss -9.607
+Valid Summary | End of Epoch 35 | Time 611.96s | Valid Loss -8.786
+Test Summary | End of Epoch 35 | Time 304.18s | Test Loss -9.703
+Train Summary | End of Epoch 36 | Time 10662.46s | Train Loss -9.655
+Valid Summary | End of Epoch 36 | Time 611.76s | Valid Loss -8.833
+Test Summary | End of Epoch 36 | Time 304.26s | Test Loss -9.735
+Fund new best model, dict saved
+Train Summary | End of Epoch 37 | Time 10662.54s | Train Loss -9.721
+Valid Summary | End of Epoch 37 | Time 612.18s | Valid Loss -8.914
+Test Summary | End of Epoch 37 | Time 304.19s | Test Loss -9.810
+Fund new best model, dict saved
+Train Summary | End of Epoch 38 | Time 10663.13s | Train Loss -9.772
+Valid Summary | End of Epoch 38 | Time 612.06s | Valid Loss -8.902
+Test Summary | End of Epoch 38 | Time 304.36s | Test Loss -9.759
+Train Summary | End of Epoch 39 | Time 10661.24s | Train Loss -9.825
+Valid Summary | End of Epoch 39 | Time 611.75s | Valid Loss -8.974
+Test Summary | End of Epoch 39 | Time 304.21s | Test Loss -9.806
+Fund new best model, dict saved
+Train Summary | End of Epoch 40 | Time 10662.98s | Train Loss -9.878
+Valid Summary | End of Epoch 40 | Time 612.25s | Valid Loss -8.918
+Test Summary | End of Epoch 40 | Time 304.14s | Test Loss -9.836
+Train Summary | End of Epoch 41 | Time 10662.94s | Train Loss -9.919
+Valid Summary | End of Epoch 41 | Time 612.15s | Valid Loss -8.889
+Test Summary | End of Epoch 41 | Time 304.30s | Test Loss -9.874
+Train Summary | End of Epoch 42 | Time 10665.59s | Train Loss -9.980
+Valid Summary | End of Epoch 42 | Time 612.08s | Valid Loss -8.965
+Test Summary | End of Epoch 42 | Time 304.16s | Test Loss -9.857
+Train Summary | End of Epoch 43 | Time 10665.85s | Train Loss -10.028
+Valid Summary | End of Epoch 43 | Time 611.91s | Valid Loss -8.998
+Test Summary | End of Epoch 43 | Time 304.24s | Test Loss -9.924
+Fund new best model, dict saved
+Train Summary | End of Epoch 44 | Time 10424.67s | Train Loss -10.076
+Valid Summary | End of Epoch 44 | Time 610.11s | Valid Loss -8.930
+Test Summary | End of Epoch 44 | Time 303.99s | Test Loss -9.907
+Train Summary | End of Epoch 45 | Time 10422.39s | Train Loss -10.107
+Valid Summary | End of Epoch 45 | Time 609.72s | Valid Loss -8.909
+Test Summary | End of Epoch 45 | Time 303.88s | Test Loss -9.927
+Train Summary | End of Epoch 46 | Time 10421.71s | Train Loss -10.158
+Valid Summary | End of Epoch 46 | Time 610.10s | Valid Loss -9.059
+Test Summary | End of Epoch 46 | Time 304.16s | Test Loss -10.024
+Fund new best model, dict saved
+Train Summary | End of Epoch 47 | Time 10419.35s | Train Loss -10.200
+Valid Summary | End of Epoch 47 | Time 610.06s | Valid Loss -9.029
+Test Summary | End of Epoch 47 | Time 303.96s | Test Loss -10.083
+Train Summary | End of Epoch 48 | Time 10439.37s | Train Loss -10.240
+Valid Summary | End of Epoch 48 | Time 610.23s | Valid Loss -9.088
+Test Summary | End of Epoch 48 | Time 303.97s | Test Loss -10.094
+Fund new best model, dict saved
+Train Summary | End of Epoch 49 | Time 10449.91s | Train Loss -10.280
+Valid Summary | End of Epoch 49 | Time 609.81s | Valid Loss -9.062
+Test Summary | End of Epoch 49 | Time 303.80s | Test Loss -10.063
+Train Summary | End of Epoch 50 | Time 10446.89s | Train Loss -10.315
+Valid Summary | End of Epoch 50 | Time 610.27s | Valid Loss -9.149
+Test Summary | End of Epoch 50 | Time 303.96s | Test Loss -10.108
+Fund new best model, dict saved
+Train Summary | End of Epoch 51 | Time 10451.47s | Train Loss -10.340
+Valid Summary | End of Epoch 51 | Time 609.86s | Valid Loss -9.030
+Test Summary | End of Epoch 51 | Time 303.94s | Test Loss -10.115
+Train Summary | End of Epoch 52 | Time 10450.57s | Train Loss -10.388
+Valid Summary | End of Epoch 52 | Time 610.34s | Valid Loss -9.155
+Test Summary | End of Epoch 52 | Time 303.96s | Test Loss -10.158
+Fund new best model, dict saved
+Train Summary | End of Epoch 53 | Time 10449.92s | Train Loss -10.428
+Valid Summary | End of Epoch 53 | Time 610.19s | Valid Loss -9.224
+Test Summary | End of Epoch 53 | Time 304.09s | Test Loss -10.183
+Fund new best model, dict saved
+Train Summary | End of Epoch 54 | Time 10447.21s | Train Loss -10.393
+Valid Summary | End of Epoch 54 | Time 610.14s | Valid Loss -9.138
+Test Summary | End of Epoch 54 | Time 303.98s | Test Loss -9.971
+Train Summary | End of Epoch 55 | Time 10445.48s | Train Loss -10.387
+Valid Summary | End of Epoch 55 | Time 610.25s | Valid Loss -9.124
+Test Summary | End of Epoch 55 | Time 303.95s | Test Loss -10.155
+Train Summary | End of Epoch 56 | Time 10445.60s | Train Loss -10.454
+Valid Summary | End of Epoch 56 | Time 610.28s | Valid Loss -9.191
+Test Summary | End of Epoch 56 | Time 304.00s | Test Loss -10.171
+Train Summary | End of Epoch 57 | Time 10448.32s | Train Loss -10.513
+Valid Summary | End of Epoch 57 | Time 610.21s | Valid Loss -9.220
+Test Summary | End of Epoch 57 | Time 304.01s | Test Loss -10.244
+Train Summary | End of Epoch 58 | Time 10447.46s | Train Loss -10.552
+Valid Summary | End of Epoch 58 | Time 610.24s | Valid Loss -9.194
+Test Summary | End of Epoch 58 | Time 303.98s | Test Loss -10.193
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000250
+Train Summary | End of Epoch 59 | Time 10452.01s | Train Loss -10.697
+Valid Summary | End of Epoch 59 | Time 610.06s | Valid Loss -9.388
+Test Summary | End of Epoch 59 | Time 304.16s | Test Loss -10.339
+Fund new best model, dict saved
+Train Summary | End of Epoch 60 | Time 10449.84s | Train Loss -10.795
+Valid Summary | End of Epoch 60 | Time 609.93s | Valid Loss -9.292
+Test Summary | End of Epoch 60 | Time 303.94s | Test Loss -10.374
+Train Summary | End of Epoch 61 | Time 10447.76s | Train Loss -10.856
+Valid Summary | End of Epoch 61 | Time 610.40s | Valid Loss -9.381
+Test Summary | End of Epoch 61 | Time 304.09s | Test Loss -10.382
+Train Summary | End of Epoch 62 | Time 10448.63s | Train Loss -10.906
+Valid Summary | End of Epoch 62 | Time 610.24s | Valid Loss -9.351
+Test Summary | End of Epoch 62 | Time 304.06s | Test Loss -10.372
+Train Summary | End of Epoch 63 | Time 10450.52s | Train Loss -10.956
+Valid Summary | End of Epoch 63 | Time 610.42s | Valid Loss -9.362
+Test Summary | End of Epoch 63 | Time 304.03s | Test Loss -10.362
+Train Summary | End of Epoch 64 | Time 10450.50s | Train Loss -10.990
+Valid Summary | End of Epoch 64 | Time 610.12s | Valid Loss -9.318
+Test Summary | End of Epoch 64 | Time 304.12s | Test Loss -10.371
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000125
+Train Summary | End of Epoch 65 | Time 10450.03s | Train Loss -10.885
+Valid Summary | End of Epoch 65 | Time 609.95s | Valid Loss -9.396
+Test Summary | End of Epoch 65 | Time 303.90s | Test Loss -10.418
+Fund new best model, dict saved
+Train Summary | End of Epoch 66 | Time 10449.01s | Train Loss -10.948
+Valid Summary | End of Epoch 66 | Time 610.45s | Valid Loss -9.409
+Test Summary | End of Epoch 66 | Time 303.96s | Test Loss -10.412
+Fund new best model, dict saved
+Train Summary | End of Epoch 67 | Time 10448.14s | Train Loss -11.002
+Valid Summary | End of Epoch 67 | Time 610.08s | Valid Loss -9.367
+Test Summary | End of Epoch 67 | Time 304.11s | Test Loss -10.416
+Train Summary | End of Epoch 68 | Time 10452.36s | Train Loss -11.047
+Valid Summary | End of Epoch 68 | Time 610.20s | Valid Loss -9.337
+Test Summary | End of Epoch 68 | Time 304.06s | Test Loss -10.386
+Train Summary | End of Epoch 69 | Time 10450.39s | Train Loss -11.080
+Valid Summary | End of Epoch 69 | Time 610.41s | Valid Loss -9.371
+Test Summary | End of Epoch 69 | Time 304.09s | Test Loss -10.388
+Train Summary | End of Epoch 70 | Time 10453.44s | Train Loss -11.110
+Valid Summary | End of Epoch 70 | Time 610.19s | Valid Loss -9.376
+Test Summary | End of Epoch 70 | Time 303.94s | Test Loss -10.408
+Train Summary | End of Epoch 71 | Time 10448.80s | Train Loss -11.143
+Valid Summary | End of Epoch 71 | Time 610.22s | Valid Loss -9.392
+Test Summary | End of Epoch 71 | Time 304.20s | Test Loss -10.402
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000063
+Train Summary | End of Epoch 72 | Time 10449.24s | Train Loss -11.037
+Valid Summary | End of Epoch 72 | Time 610.13s | Valid Loss -9.400
+Test Summary | End of Epoch 72 | Time 304.13s | Test Loss -10.421
+Train Summary | End of Epoch 73 | Time 10454.82s | Train Loss -11.073
+Valid Summary | End of Epoch 73 | Time 609.99s | Valid Loss -9.402
+Test Summary | End of Epoch 73 | Time 303.91s | Test Loss -10.436
+Train Summary | End of Epoch 74 | Time 10455.29s | Train Loss -11.108
+Valid Summary | End of Epoch 74 | Time 610.10s | Valid Loss -9.418
+Test Summary | End of Epoch 74 | Time 303.96s | Test Loss -10.421
+Fund new best model, dict saved
+Start evaluation
+Avg SISNR:i tensor([14.9690], device='cuda:0')
+Avg SNRi: 15.427827702003633
+Avg STOIi: 0.3833540046412457

checkpoints/log_LRS2_lip_tfgridnet_3spk/tensorboard/events.out.tfevents.1727769243.bach-gpu011017044238.na61.52651.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4375c5c910c8025b8c5aaad54636192b2df2bfd150f1ca6f1e18734170ffbedd
+size 6452

checkpoints/log_LRS2_lip_tfgridnet_3spk/tensorboard/events.out.tfevents.1728269430.nls-dev-servers011167134195.na63.96142.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6d3f458b99a5612988dc77a21ad6e5ef8e1c63cc0149dc1072eafc32b8d64d6
+size 4676

checkpoints/log_LRS2_lip_tfgridnet_3spk/tensorboard/events.out.tfevents.1728630190.dsw-111903-69597c964d-glc75.604215.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cb2350eead52fb6f716685cbed7ea6f23578dd1a129dac94fa57c1b2bd1e0a2
+size 88