Upload 6 files

checkpoints/log_VoxCeleb2_lip_dprnn_2spk/config.yaml

@@ -0,0 +1,42 @@
+## Config file
+
+# Log 
+seed: 777
+use_cuda: 1           # 1 for True, 0 for False
+
+# dataset
+speaker_no: 2
+mix_lst_path: ./data/VoxCeleb2/mixture_data_list_2mix.csv
+audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/
+reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/
+audio_sr: 16000
+ref_sr: 25
+
+# dataloader
+num_workers: 4
+batch_size: 8         # 2-GPU training with a total effective batch size of 16
+accu_grad: 0
+effec_batch_size: 4   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
+max_length: 6         # truncate the utterances in dataloader, in seconds 
+
+# network settings
+init_from: None       # 'None' or a log name 'log_2024-07-22(18:12:13)'
+causal: 0             # 1 for True, 0 for False
+network_reference:
+  cue: lip            # lip or speech or gesture or EEG
+  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
+  emb_size: 256       # resnet18:256
+network_audio:
+  backbone: av_dprnn
+  N: 256
+  L: 40
+  B: 64
+  H: 128
+  K: 100
+  R: 6
+
+# optimizer
+loss_type: sisdr      # "snr", "sisdr", "hybrid"
+init_learning_rate: 0.001
+max_epoch: 150
+clip_grad_norm: 5

checkpoints/log_VoxCeleb2_lip_dprnn_2spk/last_best_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98e529110aada7576f4ac360ee1d4c338dc63fdd805d52716129754f26b4f1b4
+size 94590482

checkpoints/log_VoxCeleb2_lip_dprnn_2spk/last_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4732a58efa78cc0aca08dcb9ad9ba121dfb32b9bb8886575faea19ece92afda
+size 94585962

checkpoints/log_VoxCeleb2_lip_dprnn_2spk/log_2024-10-16(15:27:35).txt

@@ -0,0 +1,611 @@
+## Config file
+
+# Log 
+seed: 777
+use_cuda: 1           # 1 for True, 0 for False
+
+# dataset
+speaker_no: 2
+mix_lst_path: ./data/VoxCeleb2/mixture_data_list_2mix.csv
+audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/
+reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/
+audio_sr: 16000
+ref_sr: 25
+
+# dataloader
+num_workers: 4
+batch_size: 8         # 2-GPU training with a total effective batch size of 16
+accu_grad: 0
+effec_batch_size: 4   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
+max_length: 6         # truncate the utterances in dataloader, in seconds 
+
+# network settings
+init_from: None       # 'None' or a log name 'log_2024-07-22(18:12:13)'
+causal: 0             # 1 for True, 0 for False
+network_reference:
+  cue: lip            # lip or speech or gesture or EEG
+  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
+  emb_size: 256       # resnet18:256
+network_audio:
+  backbone: dprnn
+  N: 256
+  L: 40
+  B: 64
+  H: 128
+  K: 100
+  R: 6
+
+# optimizer
+loss_type: sisdr      # "snr", "sisdr", "hybrid"
+init_learning_rate: 0.001
+max_epoch: 150
+clip_grad_norm: 5
+W1016 15:28:05.450172 140099065915200 torch/distributed/run.py:779] 
+W1016 15:28:05.450172 140099065915200 torch/distributed/run.py:779] *****************************************
+W1016 15:28:05.450172 140099065915200 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W1016 15:28:05.450172 140099065915200 torch/distributed/run.py:779] *****************************************
+started on checkpoints/log_2024-10-16(15:27:34)
+
+namespace(accu_grad=0, audio_direc='/mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/', audio_sr=16000, batch_size=8, causal=0, checkpoint_dir='checkpoints/log_2024-10-16(15:27:34)', clip_grad_norm=5.0, config=[<yamlargparse.Path object at 0x7f335406e640>], device=device(type='cuda'), distributed=True, effec_batch_size=4, init_from='None', init_learning_rate=0.001, local_rank=0, loss_type='sisdr', lr_warmup=0, max_epoch=150, max_length=6, mix_lst_path='./data/VoxCeleb2/mixture_data_list_2mix.csv', network_audio=namespace(B=64, H=128, K=100, L=40, N=256, R=6, backbone='dprnn'), network_reference=namespace(backbone='resnet18', cue='lip', emb_size=256), num_workers=4, ref_sr=25, reference_direc='/mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/', seed=777, speaker_no=2, train_from_last_checkpoint=0, use_cuda=1, world_size=2)
+network_wrapper(
+  (sep_network): Dprnn(
+    (encoder): Encoder(
+      (conv1d_U): Conv1d(1, 256, kernel_size=(40,), stride=(20,), bias=False)
+    )
+    (separator): rnn(
+      (layer_norm): GroupNorm(1, 256, eps=1e-08, affine=True)
+      (bottleneck_conv1x1): Conv1d(256, 64, kernel_size=(1,), stride=(1,), bias=False)
+      (dual_rnn): ModuleList(
+        (0-5): 6 x Dual_RNN_Block(
+          (intra_rnn): LSTM(64, 128, batch_first=True, bidirectional=True)
+          (inter_rnn): LSTM(64, 128, batch_first=True, bidirectional=True)
+          (intra_norm): GroupNorm(1, 64, eps=1e-08, affine=True)
+          (inter_norm): GroupNorm(1, 64, eps=1e-08, affine=True)
+          (intra_linear): Linear(in_features=256, out_features=64, bias=True)
+          (inter_linear): Linear(in_features=256, out_features=64, bias=True)
+        )
+      )
+      (prelu): PReLU(num_parameters=1)
+      (mask_conv1x1): Conv1d(64, 256, kernel_size=(1,), stride=(1,), bias=False)
+      (av_conv): Conv1d(320, 64, kernel_size=(1,), stride=(1,), bias=False)
+    )
+    (decoder): Decoder(
+      (basis_signals): Linear(in_features=256, out_features=40, bias=False)
+    )
+  )
+  (ref_encoder): Visual_encoder(
+    (v_frontend): VisualFrontend(
+      (frontend3D): Sequential(
+        (0): Conv3d(1, 64, kernel_size=(5, 7, 7), stride=(1, 2, 2), padding=(2, 3, 3), bias=False)
+        (1): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        (2): ReLU()
+        (3): MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1), dilation=1, ceil_mode=False)
+      )
+      (resnet): ResNet(
+        (layer1): ResNetLayer(
+          (conv1a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
+          (outbna): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer2): ResNetLayer(
+          (conv1a): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer3): ResNetLayer(
+          (conv1a): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer4): ResNetLayer(
+          (conv1a): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (avgpool): AvgPool2d(kernel_size=(4, 4), stride=(1, 1), padding=0)
+      )
+    )
+    (v_ds): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+    (visual_conv): Sequential(
+      (0): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (1): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (2): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (3): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (4): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+    )
+  )
+)
+
+Total number of parameters: 15306950 
+
+
+Total number of trainable parameters: 4121862 
+
+dlc1xpmyvbppmvru-master-0:29:29 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1xpmyvbppmvru-master-0:29:29 [0] NCCL INFO Bootstrap : Using eth0:22.3.234.0<0>
+dlc1xpmyvbppmvru-master-0:29:29 [0] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1xpmyvbppmvru-master-0:29:29 [0] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
+dlc1xpmyvbppmvru-master-0:29:29 [0] NCCL INFO cudaDriverVersion 11040
+dlc1xpmyvbppmvru-master-0:30:30 [1] NCCL INFO cudaDriverVersion 11040
+NCCL version 2.20.5+cuda11.8
+dlc1xpmyvbppmvru-master-0:30:30 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1xpmyvbppmvru-master-0:30:30 [1] NCCL INFO Bootstrap : Using eth0:22.3.234.0<0>
+dlc1xpmyvbppmvru-master-0:30:30 [1] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1xpmyvbppmvru-master-0:30:30 [1] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO NCCL_IB_HCA set to mlx5
+libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [RO]; OOB eth0:22.3.234.0<0>
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO Using non-device net plugin version 0
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO Using network IB
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [RO]; OOB eth0:22.3.234.0<0>
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Using non-device net plugin version 0
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Using network IB
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO comm 0x9335780 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 10 commId 0x910afc8f76702aa2 - Init START
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO comm 0x8a4edc0 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 20 commId 0x910afc8f76702aa2 - Init START
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Setting affinity for GPU 0 to 0fff
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO Setting affinity for GPU 1 to 0fff
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO comm 0x9335780 rank 0 nRanks 2 nNodes 1 localRanks 2 localRank 0 MNNVL 0
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO comm 0x8a4edc0 rank 1 nRanks 2 nNodes 1 localRanks 2 localRank 1 MNNVL 0
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Channel 00/04 :    0   1
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Channel 01/04 :    0   1
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Channel 02/04 :    0   1
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO Trees [0] -1/-1/-1->1->0 [1] 0/-1/-1->1->-1 [2] -1/-1/-1->1->0 [3] 0/-1/-1->1->-1
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Channel 03/04 :    0   1
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO P2P Chunksize set to 524288
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] -1/-1/-1->0->1 [2] 1/-1/-1->0->-1 [3] -1/-1/-1->0->1
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO P2P Chunksize set to 524288
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO Channel 01/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO Channel 03/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Connected all rings
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO Connected all trees
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO Connected all rings
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO Connected all trees
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO 4 coll channels, 0 collnet channels, 0 nvls channels, 4 p2p channels, 4 p2p channels per peer
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO 4 coll channels, 0 collnet channels, 0 nvls channels, 4 p2p channels, 4 p2p channels per peer
+dlc1xpmyvbppmvru-master-0:29:46 [0] NCCL INFO comm 0x9335780 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 10 commId 0x910afc8f76702aa2 - Init COMPLETE
+dlc1xpmyvbppmvru-master-0:30:47 [1] NCCL INFO comm 0x8a4edc0 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 20 commId 0x910afc8f76702aa2 - Init COMPLETE
+Start new training from scratch
+[rank0]:[W1016 15:29:26.567341744 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank1]:[W1016 15:29:26.567396715 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+Train Summary | End of Epoch 1 | Time 1136.03s | Train Loss -1.143
+Valid Summary | End of Epoch 1 | Time 186.54s | Valid Loss -1.961
+Test Summary | End of Epoch 1 | Time 89.69s | Test Loss -1.949
+Fund new best model, dict saved
+Train Summary | End of Epoch 2 | Time 532.39s | Train Loss -2.563
+Valid Summary | End of Epoch 2 | Time 62.43s | Valid Loss -3.072
+Test Summary | End of Epoch 2 | Time 36.68s | Test Loss -3.038
+Fund new best model, dict saved
+Train Summary | End of Epoch 3 | Time 522.50s | Train Loss -3.681
+Valid Summary | End of Epoch 3 | Time 61.41s | Valid Loss -3.867
+Test Summary | End of Epoch 3 | Time 34.41s | Test Loss -3.704
+Fund new best model, dict saved
+Train Summary | End of Epoch 4 | Time 521.11s | Train Loss -4.505
+Valid Summary | End of Epoch 4 | Time 60.64s | Valid Loss -4.717
+Test Summary | End of Epoch 4 | Time 34.68s | Test Loss -4.602
+Fund new best model, dict saved
+Train Summary | End of Epoch 5 | Time 520.90s | Train Loss -5.112
+Valid Summary | End of Epoch 5 | Time 60.13s | Valid Loss -5.280
+Test Summary | End of Epoch 5 | Time 34.57s | Test Loss -5.177
+Fund new best model, dict saved
+Train Summary | End of Epoch 6 | Time 527.34s | Train Loss -5.567
+Valid Summary | End of Epoch 6 | Time 60.35s | Valid Loss -5.635
+Test Summary | End of Epoch 6 | Time 35.99s | Test Loss -5.481
+Fund new best model, dict saved
+Train Summary | End of Epoch 7 | Time 524.58s | Train Loss -5.973
+Valid Summary | End of Epoch 7 | Time 64.03s | Valid Loss -6.066
+Test Summary | End of Epoch 7 | Time 35.53s | Test Loss -5.906
+Fund new best model, dict saved
+Train Summary | End of Epoch 8 | Time 521.69s | Train Loss -6.393
+Valid Summary | End of Epoch 8 | Time 60.44s | Valid Loss -6.381
+Test Summary | End of Epoch 8 | Time 34.78s | Test Loss -6.205
+Fund new best model, dict saved
+Train Summary | End of Epoch 9 | Time 524.26s | Train Loss -6.815
+Valid Summary | End of Epoch 9 | Time 60.49s | Valid Loss -6.840
+Test Summary | End of Epoch 9 | Time 34.27s | Test Loss -6.577
+Fund new best model, dict saved
+Train Summary | End of Epoch 10 | Time 527.01s | Train Loss -7.251
+Valid Summary | End of Epoch 10 | Time 60.70s | Valid Loss -7.199
+Test Summary | End of Epoch 10 | Time 34.01s | Test Loss -6.979
+Fund new best model, dict saved
+Train Summary | End of Epoch 11 | Time 519.57s | Train Loss -7.656
+Valid Summary | End of Epoch 11 | Time 59.87s | Valid Loss -7.626
+Test Summary | End of Epoch 11 | Time 34.53s | Test Loss -7.344
+Fund new best model, dict saved
+Train Summary | End of Epoch 12 | Time 518.91s | Train Loss -8.007
+Valid Summary | End of Epoch 12 | Time 59.72s | Valid Loss -7.927
+Test Summary | End of Epoch 12 | Time 34.57s | Test Loss -7.680
+Fund new best model, dict saved
+Train Summary | End of Epoch 13 | Time 519.78s | Train Loss -8.310
+Valid Summary | End of Epoch 13 | Time 59.64s | Valid Loss -8.179
+Test Summary | End of Epoch 13 | Time 34.45s | Test Loss -7.865
+Fund new best model, dict saved
+Train Summary | End of Epoch 14 | Time 518.22s | Train Loss -8.636
+Valid Summary | End of Epoch 14 | Time 60.08s | Valid Loss -8.400
+Test Summary | End of Epoch 14 | Time 34.13s | Test Loss -8.087
+Fund new best model, dict saved
+Train Summary | End of Epoch 15 | Time 519.43s | Train Loss -8.894
+Valid Summary | End of Epoch 15 | Time 60.15s | Valid Loss -8.782
+Test Summary | End of Epoch 15 | Time 34.21s | Test Loss -8.444
+Fund new best model, dict saved
+Train Summary | End of Epoch 16 | Time 519.03s | Train Loss -9.191
+Valid Summary | End of Epoch 16 | Time 61.51s | Valid Loss -8.883
+Test Summary | End of Epoch 16 | Time 33.95s | Test Loss -8.592
+Fund new best model, dict saved
+Train Summary | End of Epoch 17 | Time 518.84s | Train Loss -9.422
+Valid Summary | End of Epoch 17 | Time 59.83s | Valid Loss -9.109
+Test Summary | End of Epoch 17 | Time 34.50s | Test Loss -8.798
+Fund new best model, dict saved
+Train Summary | End of Epoch 18 | Time 518.31s | Train Loss -9.619
+Valid Summary | End of Epoch 18 | Time 59.61s | Valid Loss -9.177
+Test Summary | End of Epoch 18 | Time 34.34s | Test Loss -8.867
+Fund new best model, dict saved
+Train Summary | End of Epoch 19 | Time 517.49s | Train Loss -9.836
+Valid Summary | End of Epoch 19 | Time 60.48s | Valid Loss -9.359
+Test Summary | End of Epoch 19 | Time 34.82s | Test Loss -9.041
+Fund new best model, dict saved
+Train Summary | End of Epoch 20 | Time 520.49s | Train Loss -9.985
+Valid Summary | End of Epoch 20 | Time 60.91s | Valid Loss -9.508
+Test Summary | End of Epoch 20 | Time 34.80s | Test Loss -9.154
+Fund new best model, dict saved
+Train Summary | End of Epoch 21 | Time 521.76s | Train Loss -10.156
+Valid Summary | End of Epoch 21 | Time 61.59s | Valid Loss -9.648
+Test Summary | End of Epoch 21 | Time 35.30s | Test Loss -9.361
+Fund new best model, dict saved
+Train Summary | End of Epoch 22 | Time 520.25s | Train Loss -10.321
+Valid Summary | End of Epoch 22 | Time 60.19s | Valid Loss -9.891
+Test Summary | End of Epoch 22 | Time 34.65s | Test Loss -9.584
+Fund new best model, dict saved
+Train Summary | End of Epoch 23 | Time 518.41s | Train Loss -10.437
+Valid Summary | End of Epoch 23 | Time 60.06s | Valid Loss -9.806
+Test Summary | End of Epoch 23 | Time 34.87s | Test Loss -9.465
+Train Summary | End of Epoch 24 | Time 518.00s | Train Loss -10.558
+Valid Summary | End of Epoch 24 | Time 60.04s | Valid Loss -10.033
+Test Summary | End of Epoch 24 | Time 34.30s | Test Loss -9.654
+Fund new best model, dict saved
+Train Summary | End of Epoch 25 | Time 518.19s | Train Loss -10.674
+Valid Summary | End of Epoch 25 | Time 60.68s | Valid Loss -10.149
+Test Summary | End of Epoch 25 | Time 34.28s | Test Loss -9.742
+Fund new best model, dict saved
+Train Summary | End of Epoch 26 | Time 517.83s | Train Loss -10.793
+Valid Summary | End of Epoch 26 | Time 59.94s | Valid Loss -10.301
+Test Summary | End of Epoch 26 | Time 34.41s | Test Loss -9.923
+Fund new best model, dict saved
+Train Summary | End of Epoch 27 | Time 519.54s | Train Loss -10.893
+Valid Summary | End of Epoch 27 | Time 59.71s | Valid Loss -10.337
+Test Summary | End of Epoch 27 | Time 34.16s | Test Loss -9.964
+Fund new best model, dict saved
+Train Summary | End of Epoch 28 | Time 517.22s | Train Loss -10.998
+Valid Summary | End of Epoch 28 | Time 60.20s | Valid Loss -10.414
+Test Summary | End of Epoch 28 | Time 35.08s | Test Loss -9.978
+Fund new best model, dict saved
+Train Summary | End of Epoch 29 | Time 519.76s | Train Loss -11.092
+Valid Summary | End of Epoch 29 | Time 59.35s | Valid Loss -10.409
+Test Summary | End of Epoch 29 | Time 34.43s | Test Loss -10.057
+Train Summary | End of Epoch 30 | Time 519.13s | Train Loss -11.168
+Valid Summary | End of Epoch 30 | Time 59.80s | Valid Loss -10.482
+Test Summary | End of Epoch 30 | Time 34.34s | Test Loss -10.156
+Fund new best model, dict saved
+Train Summary | End of Epoch 31 | Time 519.57s | Train Loss -11.234
+Valid Summary | End of Epoch 31 | Time 59.49s | Valid Loss -10.528
+Test Summary | End of Epoch 31 | Time 34.12s | Test Loss -10.135
+Fund new best model, dict saved
+Train Summary | End of Epoch 32 | Time 519.21s | Train Loss -11.322
+Valid Summary | End of Epoch 32 | Time 60.14s | Valid Loss -10.618
+Test Summary | End of Epoch 32 | Time 34.59s | Test Loss -10.253
+Fund new best model, dict saved
+Train Summary | End of Epoch 33 | Time 518.09s | Train Loss -11.399
+Valid Summary | End of Epoch 33 | Time 59.79s | Valid Loss -10.616
+Test Summary | End of Epoch 33 | Time 34.67s | Test Loss -10.181
+Train Summary | End of Epoch 34 | Time 519.15s | Train Loss -11.461
+Valid Summary | End of Epoch 34 | Time 59.91s | Valid Loss -10.738
+Test Summary | End of Epoch 34 | Time 33.96s | Test Loss -10.300
+Fund new best model, dict saved
+Train Summary | End of Epoch 35 | Time 519.16s | Train Loss -11.516
+Valid Summary | End of Epoch 35 | Time 61.33s | Valid Loss -10.684
+Test Summary | End of Epoch 35 | Time 34.44s | Test Loss -10.158
+Train Summary | End of Epoch 36 | Time 519.31s | Train Loss -11.607
+Valid Summary | End of Epoch 36 | Time 59.58s | Valid Loss -10.844
+Test Summary | End of Epoch 36 | Time 34.49s | Test Loss -10.436
+Fund new best model, dict saved
+Train Summary | End of Epoch 37 | Time 518.54s | Train Loss -11.652
+Valid Summary | End of Epoch 37 | Time 59.47s | Valid Loss -10.818
+Test Summary | End of Epoch 37 | Time 34.26s | Test Loss -10.365
+Train Summary | End of Epoch 38 | Time 518.83s | Train Loss -11.721
+Valid Summary | End of Epoch 38 | Time 61.51s | Valid Loss -10.763
+Test Summary | End of Epoch 38 | Time 34.26s | Test Loss -10.297
+Train Summary | End of Epoch 39 | Time 519.14s | Train Loss -11.780
+Valid Summary | End of Epoch 39 | Time 60.13s | Valid Loss -10.801
+Test Summary | End of Epoch 39 | Time 34.26s | Test Loss -10.421
+Train Summary | End of Epoch 40 | Time 518.87s | Train Loss -11.832
+Valid Summary | End of Epoch 40 | Time 59.66s | Valid Loss -10.950
+Test Summary | End of Epoch 40 | Time 34.57s | Test Loss -10.544
+Fund new best model, dict saved
+Train Summary | End of Epoch 41 | Time 518.29s | Train Loss -11.874
+Valid Summary | End of Epoch 41 | Time 60.20s | Valid Loss -11.077
+Test Summary | End of Epoch 41 | Time 34.59s | Test Loss -10.605
+Fund new best model, dict saved
+Train Summary | End of Epoch 42 | Time 518.75s | Train Loss -11.927
+Valid Summary | End of Epoch 42 | Time 60.51s | Valid Loss -11.021
+Test Summary | End of Epoch 42 | Time 34.21s | Test Loss -10.710
+Train Summary | End of Epoch 43 | Time 518.21s | Train Loss -11.972
+Valid Summary | End of Epoch 43 | Time 60.24s | Valid Loss -11.148
+Test Summary | End of Epoch 43 | Time 34.35s | Test Loss -10.754
+Fund new best model, dict saved
+Train Summary | End of Epoch 44 | Time 517.93s | Train Loss -12.025
+Valid Summary | End of Epoch 44 | Time 59.79s | Valid Loss -11.016
+Test Summary | End of Epoch 44 | Time 35.07s | Test Loss -10.649
+Train Summary | End of Epoch 45 | Time 517.34s | Train Loss -12.058
+Valid Summary | End of Epoch 45 | Time 59.79s | Valid Loss -11.158
+Test Summary | End of Epoch 45 | Time 34.52s | Test Loss -10.659
+Fund new best model, dict saved
+Train Summary | End of Epoch 46 | Time 519.56s | Train Loss -12.102
+Valid Summary | End of Epoch 46 | Time 60.24s | Valid Loss -11.202
+Test Summary | End of Epoch 46 | Time 33.99s | Test Loss -10.825
+Fund new best model, dict saved
+Train Summary | End of Epoch 47 | Time 518.26s | Train Loss -12.128
+Valid Summary | End of Epoch 47 | Time 59.93s | Valid Loss -11.294
+Test Summary | End of Epoch 47 | Time 34.05s | Test Loss -10.846
+Fund new best model, dict saved
+Train Summary | End of Epoch 48 | Time 518.20s | Train Loss -12.189
+Valid Summary | End of Epoch 48 | Time 59.72s | Valid Loss -11.100
+Test Summary | End of Epoch 48 | Time 34.81s | Test Loss -10.742
+Train Summary | End of Epoch 49 | Time 520.08s | Train Loss -12.211
+Valid Summary | End of Epoch 49 | Time 60.13s | Valid Loss -11.270
+Test Summary | End of Epoch 49 | Time 34.61s | Test Loss -10.813
+Train Summary | End of Epoch 50 | Time 518.98s | Train Loss -12.259
+Valid Summary | End of Epoch 50 | Time 60.14s | Valid Loss -11.058
+Test Summary | End of Epoch 50 | Time 34.37s | Test Loss -10.562
+Train Summary | End of Epoch 51 | Time 518.45s | Train Loss -12.284
+Valid Summary | End of Epoch 51 | Time 59.56s | Valid Loss -11.306
+Test Summary | End of Epoch 51 | Time 34.70s | Test Loss -10.906
+Fund new best model, dict saved
+Train Summary | End of Epoch 52 | Time 519.03s | Train Loss -12.322
+Valid Summary | End of Epoch 52 | Time 60.26s | Valid Loss -11.384
+Test Summary | End of Epoch 52 | Time 34.40s | Test Loss -10.926
+Fund new best model, dict saved
+Train Summary | End of Epoch 53 | Time 518.84s | Train Loss -12.363
+Valid Summary | End of Epoch 53 | Time 60.39s | Valid Loss -11.297
+Test Summary | End of Epoch 53 | Time 34.45s | Test Loss -10.880
+Train Summary | End of Epoch 54 | Time 518.32s | Train Loss -12.377
+Valid Summary | End of Epoch 54 | Time 59.67s | Valid Loss -11.256
+Test Summary | End of Epoch 54 | Time 34.42s | Test Loss -10.960
+Train Summary | End of Epoch 55 | Time 518.59s | Train Loss -12.419
+Valid Summary | End of Epoch 55 | Time 59.80s | Valid Loss -10.645
+Test Summary | End of Epoch 55 | Time 34.77s | Test Loss -10.457
+Train Summary | End of Epoch 56 | Time 517.86s | Train Loss -12.440
+Valid Summary | End of Epoch 56 | Time 59.75s | Valid Loss -11.398
+Test Summary | End of Epoch 56 | Time 34.60s | Test Loss -11.014
+Fund new best model, dict saved
+Train Summary | End of Epoch 57 | Time 518.30s | Train Loss -12.484
+Valid Summary | End of Epoch 57 | Time 59.82s | Valid Loss -11.353
+Test Summary | End of Epoch 57 | Time 34.41s | Test Loss -11.011
+Train Summary | End of Epoch 58 | Time 520.20s | Train Loss -12.513
+Valid Summary | End of Epoch 58 | Time 60.28s | Valid Loss -11.232
+Test Summary | End of Epoch 58 | Time 34.11s | Test Loss -10.877
+Train Summary | End of Epoch 59 | Time 518.46s | Train Loss -12.524
+Valid Summary | End of Epoch 59 | Time 59.65s | Valid Loss -11.421
+Test Summary | End of Epoch 59 | Time 34.41s | Test Loss -11.073
+Fund new best model, dict saved
+Train Summary | End of Epoch 60 | Time 518.92s | Train Loss -12.554
+Valid Summary | End of Epoch 60 | Time 59.60s | Valid Loss -11.445
+Test Summary | End of Epoch 60 | Time 34.88s | Test Loss -11.093
+Fund new best model, dict saved
+Train Summary | End of Epoch 61 | Time 517.26s | Train Loss -12.583
+Valid Summary | End of Epoch 61 | Time 59.43s | Valid Loss -11.472
+Test Summary | End of Epoch 61 | Time 35.05s | Test Loss -11.012
+Fund new best model, dict saved
+Train Summary | End of Epoch 62 | Time 517.27s | Train Loss -12.624
+Valid Summary | End of Epoch 62 | Time 59.78s | Valid Loss -11.460
+Test Summary | End of Epoch 62 | Time 34.77s | Test Loss -11.070
+Train Summary | End of Epoch 63 | Time 518.49s | Train Loss -12.637
+Valid Summary | End of Epoch 63 | Time 60.43s | Valid Loss -11.498
+Test Summary | End of Epoch 63 | Time 34.49s | Test Loss -11.066
+Fund new best model, dict saved
+Train Summary | End of Epoch 64 | Time 518.74s | Train Loss -12.660
+Valid Summary | End of Epoch 64 | Time 59.84s | Valid Loss -11.363
+Test Summary | End of Epoch 64 | Time 34.64s | Test Loss -10.865
+Train Summary | End of Epoch 65 | Time 517.86s | Train Loss -12.674
+Valid Summary | End of Epoch 65 | Time 59.65s | Valid Loss -10.896
+Test Summary | End of Epoch 65 | Time 34.13s | Test Loss -10.534
+Train Summary | End of Epoch 66 | Time 517.95s | Train Loss -12.704
+Valid Summary | End of Epoch 66 | Time 59.73s | Valid Loss -11.279
+Test Summary | End of Epoch 66 | Time 34.65s | Test Loss -10.885
+Train Summary | End of Epoch 67 | Time 518.19s | Train Loss -12.721
+Valid Summary | End of Epoch 67 | Time 60.00s | Valid Loss -11.364
+Test Summary | End of Epoch 67 | Time 34.22s | Test Loss -10.883
+Train Summary | End of Epoch 68 | Time 517.83s | Train Loss -12.744
+Valid Summary | End of Epoch 68 | Time 60.40s | Valid Loss -11.619
+Test Summary | End of Epoch 68 | Time 34.24s | Test Loss -11.204
+Fund new best model, dict saved
+Train Summary | End of Epoch 69 | Time 519.78s | Train Loss -12.776
+Valid Summary | End of Epoch 69 | Time 60.07s | Valid Loss -11.411
+Test Summary | End of Epoch 69 | Time 34.55s | Test Loss -10.848
+Train Summary | End of Epoch 70 | Time 518.22s | Train Loss -12.801
+Valid Summary | End of Epoch 70 | Time 59.45s | Valid Loss -11.016
+Test Summary | End of Epoch 70 | Time 34.57s | Test Loss -10.476
+Train Summary | End of Epoch 71 | Time 518.52s | Train Loss -12.804
+Valid Summary | End of Epoch 71 | Time 60.45s | Valid Loss -11.440
+Test Summary | End of Epoch 71 | Time 34.61s | Test Loss -11.051
+Train Summary | End of Epoch 72 | Time 519.23s | Train Loss -12.837
+Valid Summary | End of Epoch 72 | Time 60.35s | Valid Loss -11.442
+Test Summary | End of Epoch 72 | Time 34.71s | Test Loss -10.937
+Train Summary | End of Epoch 73 | Time 520.03s | Train Loss -12.856
+Valid Summary | End of Epoch 73 | Time 59.99s | Valid Loss -11.348
+Test Summary | End of Epoch 73 | Time 34.85s | Test Loss -10.732
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000500
+Train Summary | End of Epoch 74 | Time 518.69s | Train Loss -13.019
+Valid Summary | End of Epoch 74 | Time 60.37s | Valid Loss -11.562
+Test Summary | End of Epoch 74 | Time 34.66s | Test Loss -11.118
+Train Summary | End of Epoch 75 | Time 519.74s | Train Loss -13.069
+Valid Summary | End of Epoch 75 | Time 60.23s | Valid Loss -11.680
+Test Summary | End of Epoch 75 | Time 34.85s | Test Loss -11.168
+Fund new best model, dict saved
+Train Summary | End of Epoch 76 | Time 521.19s | Train Loss -13.105
+Valid Summary | End of Epoch 76 | Time 60.98s | Valid Loss -11.147
+Test Summary | End of Epoch 76 | Time 34.75s | Test Loss -10.639
+Train Summary | End of Epoch 77 | Time 521.28s | Train Loss -13.131
+Valid Summary | End of Epoch 77 | Time 60.25s | Valid Loss -11.306
+Test Summary | End of Epoch 77 | Time 34.47s | Test Loss -10.749
+Train Summary | End of Epoch 78 | Time 521.05s | Train Loss -13.152
+Valid Summary | End of Epoch 78 | Time 59.92s | Valid Loss -11.515
+Test Summary | End of Epoch 78 | Time 34.87s | Test Loss -11.080
+Train Summary | End of Epoch 79 | Time 519.71s | Train Loss -13.175
+Valid Summary | End of Epoch 79 | Time 60.42s | Valid Loss -11.498
+Test Summary | End of Epoch 79 | Time 34.83s | Test Loss -11.020
+Train Summary | End of Epoch 80 | Time 521.39s | Train Loss -13.193
+Valid Summary | End of Epoch 80 | Time 59.97s | Valid Loss -11.811
+Test Summary | End of Epoch 80 | Time 34.58s | Test Loss -11.380
+Fund new best model, dict saved
+Train Summary | End of Epoch 81 | Time 518.44s | Train Loss -13.211
+Valid Summary | End of Epoch 81 | Time 59.77s | Valid Loss -11.735
+Test Summary | End of Epoch 81 | Time 34.51s | Test Loss -11.326
+Train Summary | End of Epoch 82 | Time 520.08s | Train Loss -13.234
+Valid Summary | End of Epoch 82 | Time 60.43s | Valid Loss -11.280
+Test Summary | End of Epoch 82 | Time 34.64s | Test Loss -10.932
+Train Summary | End of Epoch 83 | Time 519.69s | Train Loss -13.246
+Valid Summary | End of Epoch 83 | Time 60.50s | Valid Loss -11.337
+Test Summary | End of Epoch 83 | Time 35.30s | Test Loss -10.917
+Train Summary | End of Epoch 84 | Time 519.87s | Train Loss -13.262
+Valid Summary | End of Epoch 84 | Time 60.09s | Valid Loss -11.391
+Test Summary | End of Epoch 84 | Time 34.72s | Test Loss -11.022
+Train Summary | End of Epoch 85 | Time 521.15s | Train Loss -13.277
+Valid Summary | End of Epoch 85 | Time 60.43s | Valid Loss -11.435
+Test Summary | End of Epoch 85 | Time 34.72s | Test Loss -11.165
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000250
+Train Summary | End of Epoch 86 | Time 518.64s | Train Loss -13.314
+Valid Summary | End of Epoch 86 | Time 59.52s | Valid Loss -11.558
+Test Summary | End of Epoch 86 | Time 34.45s | Test Loss -11.134
+Train Summary | End of Epoch 87 | Time 518.60s | Train Loss -13.337
+Valid Summary | End of Epoch 87 | Time 59.34s | Valid Loss -11.595
+Test Summary | End of Epoch 87 | Time 34.35s | Test Loss -11.278
+Train Summary | End of Epoch 88 | Time 518.53s | Train Loss -13.356
+Valid Summary | End of Epoch 88 | Time 59.97s | Valid Loss -11.571
+Test Summary | End of Epoch 88 | Time 34.59s | Test Loss -11.095
+Train Summary | End of Epoch 89 | Time 518.29s | Train Loss -13.369
+Valid Summary | End of Epoch 89 | Time 59.97s | Valid Loss -11.070
+Test Summary | End of Epoch 89 | Time 33.97s | Test Loss -10.587
+Train Summary | End of Epoch 90 | Time 518.67s | Train Loss -13.385
+Valid Summary | End of Epoch 90 | Time 59.42s | Valid Loss -11.527
+Test Summary | End of Epoch 90 | Time 33.97s | Test Loss -11.091
+No imporvement for 10 epochs, early stopping.
+Start evaluation
+Avg SISNR:i tensor([11.4575], device='cuda:0')
+Avg SNRi: 11.7811785187395
+Avg PESQi: 0.8883232574065526
+Avg STOIi: 0.23720669982369647

checkpoints/log_VoxCeleb2_lip_dprnn_2spk/tensorboard/events.out.tfevents.1729063739.dlc1xpmyvbppmvru-master-0.29.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9b0414019fdd7e35c60240876a2de76cd7ff8182096987efd75d6f749c38b77
+size 13260