Model save

README.md

@@ -16,15 +16,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5987
-- Rewards/chosen: -0.6320
-- Rewards/rejected: -1.1187
-- Rewards/accuracies: 0.7070
-- Rewards/margins: 0.4867
-- Logps/rejected: -1395.8809
-- Logps/chosen: -907.7229
-- Logits/rejected: -0.5772
-- Logits/chosen: -0.6368
+- Loss: 0.6172
+- Rewards/chosen: -0.4655
+- Rewards/rejected: -0.7989
+- Rewards/accuracies: 0.75
+- Rewards/margins: 0.3335
+- Logps/rejected: -1076.8990
+- Logps/chosen: -740.6094
+- Logits/rejected: -0.8340
+- Logits/chosen: -0.8622
 
 ## Model description
 
@@ -61,7 +61,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5855        | 0.9984 | 477  | 0.5987          | -0.6320        | -1.1187          | 0.7070             | 0.4867          | -1395.8809     | -907.7229    | -0.5772         | -0.6368       |
+| 0.6047        | 0.9984 | 477  | 0.6172          | -0.4655        | -0.7989          | 0.75               | 0.3335          | -1076.8990     | -740.6094    | -0.8340         | -0.8622       |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.6339174956395691,
-    "train_runtime": 13552.9806,
+    "train_loss": 0.6437127205310877,
+    "train_runtime": 13588.3404,
     "train_samples": 61134,
-    "train_samples_per_second": 4.511,
+    "train_samples_per_second": 4.499,
     "train_steps_per_second": 0.035
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/scratch/pawsey0935/tliu1/alignment-handbook/data/zephyr-8b-sft-1e/checkpoint-200",
+  "_name_or_path": "/scratch/pawsey0935/tliu1/alignment-handbook/data/zephyr-8b-sft-1e/checkpoint-500",
   "architectures": [
     "LlamaForCausalLM"
   ],

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78365fa3381a8af16010371ce29b8dfa478d52cb1bfab568d075dca246eacded
+oid sha256:f57f7d4d01531c960492b69349323e79d4f1ef8b1e4f17132ee8910ea6aa00aa
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41668989e79c17a1de048d670ee14ca7f1004e6b5f52c4e3b1e351a002135936
+oid sha256:9a378753606915dbc7f6fad865ae1ac46232454da1d3053dad2e0bf172229c27
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60c4d41e6f5e9de2a44e720b15af055f99d3b9ec6e3759d215c9b8b209fbb1ef
+oid sha256:aa19495c5134915d35e7027ccafd563f0dce9a2650b1db3049af49b6069f63c7
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fbb846ebf90ebc765d24b01007039ab25e0c43ea0f6a7a7865efba4ef9dac2dd
+oid sha256:ac5f251487a2101a2dd9ce3b7e050ab2abd396451140fb470ac84dca406138de
 size 1168138808

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.6339174956395691,
-    "train_runtime": 13552.9806,
+    "train_loss": 0.6437127205310877,
+    "train_runtime": 13588.3404,
     "train_samples": 61134,
-    "train_samples_per_second": 4.511,
+    "train_samples_per_second": 4.499,
     "train_steps_per_second": 0.035
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0020931449502878076,
-      "grad_norm": 0.3233749908780064,
+      "grad_norm": 0.3180850451895604,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -0.9934180974960327,
-      "logits/rejected": -0.9732793569564819,
-      "logps/chosen": -362.2581481933594,
-      "logps/rejected": -361.5176696777344,
+      "logits/chosen": -1.2097231149673462,
+      "logits/rejected": -1.2087113857269287,
+      "logps/chosen": -360.4364013671875,
+      "logps/rejected": -362.17816162109375,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,732 +25,732 @@
     },
     {
       "epoch": 0.020931449502878074,
-      "grad_norm": 0.31818436563386754,
+      "grad_norm": 0.3091562668837711,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -0.9403506517410278,
-      "logits/rejected": -0.920226514339447,
-      "logps/chosen": -300.6136474609375,
-      "logps/rejected": -277.24029541015625,
+      "logits/chosen": -1.1554023027420044,
+      "logits/rejected": -1.1447501182556152,
+      "logps/chosen": -302.14666748046875,
+      "logps/rejected": -278.4107360839844,
       "loss": 0.6931,
-      "rewards/accuracies": 0.4583333432674408,
-      "rewards/chosen": 3.990698678535409e-05,
-      "rewards/margins": 2.8463546186685562e-05,
-      "rewards/rejected": 1.144344059866853e-05,
+      "rewards/accuracies": 0.4930555522441864,
+      "rewards/chosen": 7.456003459083149e-06,
+      "rewards/margins": 3.9713955629849806e-05,
+      "rewards/rejected": -3.225794353056699e-05,
       "step": 10
     },
     {
       "epoch": 0.04186289900575615,
-      "grad_norm": 0.32066911516768193,
+      "grad_norm": 0.29442577149940785,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -0.9091697931289673,
-      "logits/rejected": -0.8898228406906128,
-      "logps/chosen": -313.4816589355469,
-      "logps/rejected": -279.7200012207031,
+      "logits/chosen": -1.1197772026062012,
+      "logits/rejected": -1.1169277429580688,
+      "logps/chosen": -313.89813232421875,
+      "logps/rejected": -281.2487487792969,
       "loss": 0.6931,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": 9.299140947405249e-05,
-      "rewards/margins": 0.00010705796012189239,
-      "rewards/rejected": -1.4066553376324009e-05,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 3.8420424971263856e-05,
+      "rewards/margins": -3.498773367027752e-05,
+      "rewards/rejected": 7.340816227952018e-05,
       "step": 20
     },
     {
       "epoch": 0.06279434850863422,
-      "grad_norm": 0.29725853737386143,
+      "grad_norm": 0.2970425624012531,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -0.9132905006408691,
-      "logits/rejected": -0.9013525247573853,
-      "logps/chosen": -290.26397705078125,
-      "logps/rejected": -249.2030792236328,
+      "logits/chosen": -1.1332579851150513,
+      "logits/rejected": -1.1339633464813232,
+      "logps/chosen": -291.05224609375,
+      "logps/rejected": -250.421142578125,
       "loss": 0.6931,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0005759169580414891,
-      "rewards/margins": 0.00012212179717607796,
-      "rewards/rejected": 0.0004537951317615807,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.0005964103038422763,
+      "rewards/margins": 0.0001513070019427687,
+      "rewards/rejected": 0.0004451033310033381,
       "step": 30
     },
     {
       "epoch": 0.0837257980115123,
-      "grad_norm": 0.30422594886075766,
+      "grad_norm": 0.29488203891244075,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -0.9063602685928345,
-      "logits/rejected": -0.8846728205680847,
-      "logps/chosen": -258.3272705078125,
-      "logps/rejected": -258.79901123046875,
-      "loss": 0.6928,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.0017278010491281748,
-      "rewards/margins": 0.0005969521589577198,
-      "rewards/rejected": 0.0011308487737551332,
+      "logits/chosen": -1.1262879371643066,
+      "logits/rejected": -1.118971347808838,
+      "logps/chosen": -259.94232177734375,
+      "logps/rejected": -261.27496337890625,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.0016647384036332369,
+      "rewards/margins": 0.000596812751609832,
+      "rewards/rejected": 0.001067925593815744,
       "step": 40
     },
     {
       "epoch": 0.10465724751439037,
-      "grad_norm": 0.3141653560965076,
+      "grad_norm": 0.3058449366923936,
       "learning_rate": 4.999731868769026e-07,
-      "logits/chosen": -0.888303279876709,
-      "logits/rejected": -0.8725374937057495,
-      "logps/chosen": -286.4285583496094,
-      "logps/rejected": -271.14837646484375,
-      "loss": 0.6924,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.0034768437035381794,
-      "rewards/margins": 0.0013709357008337975,
-      "rewards/rejected": 0.002105908002704382,
+      "logits/chosen": -1.1080738306045532,
+      "logits/rejected": -1.1030961275100708,
+      "logps/chosen": -287.21307373046875,
+      "logps/rejected": -272.30548095703125,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.0033532450906932354,
+      "rewards/margins": 0.0014802367659285665,
+      "rewards/rejected": 0.001873008324764669,
       "step": 50
     },
     {
       "epoch": 0.12558869701726844,
-      "grad_norm": 0.32864602203785215,
+      "grad_norm": 0.32524143765136054,
       "learning_rate": 4.990353313429303e-07,
-      "logits/chosen": -0.8980317115783691,
-      "logits/rejected": -0.883760929107666,
-      "logps/chosen": -252.4823760986328,
-      "logps/rejected": -242.98117065429688,
-      "loss": 0.6918,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.0057683298364281654,
-      "rewards/margins": 0.003004656871780753,
-      "rewards/rejected": 0.002763673197478056,
+      "logits/chosen": -1.1249278783798218,
+      "logits/rejected": -1.1207427978515625,
+      "logps/chosen": -252.5863494873047,
+      "logps/rejected": -243.4455108642578,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.006057645194232464,
+      "rewards/margins": 0.002920362865552306,
+      "rewards/rejected": 0.0031372823286801577,
       "step": 60
     },
     {
       "epoch": 0.14652014652014653,
-      "grad_norm": 0.30851772421969176,
+      "grad_norm": 0.2965997309874686,
       "learning_rate": 4.967625656594781e-07,
-      "logits/chosen": -0.8898143768310547,
-      "logits/rejected": -0.877031683921814,
-      "logps/chosen": -293.1795959472656,
-      "logps/rejected": -282.2380676269531,
-      "loss": 0.6909,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.005099158734083176,
-      "rewards/margins": 0.004411149770021439,
-      "rewards/rejected": 0.0006880092551000416,
+      "logits/chosen": -1.124455451965332,
+      "logits/rejected": -1.1197750568389893,
+      "logps/chosen": -292.89508056640625,
+      "logps/rejected": -281.67022705078125,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.006117576267570257,
+      "rewards/margins": 0.004100095480680466,
+      "rewards/rejected": 0.0020174814853817225,
       "step": 70
     },
     {
       "epoch": 0.1674515960230246,
-      "grad_norm": 0.3603242703891539,
+      "grad_norm": 0.3681249264262344,
       "learning_rate": 4.93167072587771e-07,
-      "logits/chosen": -0.9251786470413208,
-      "logits/rejected": -0.8793746829032898,
-      "logps/chosen": -325.86688232421875,
-      "logps/rejected": -257.28363037109375,
+      "logits/chosen": -1.1383342742919922,
+      "logits/rejected": -1.123975396156311,
+      "logps/chosen": -325.63470458984375,
+      "logps/rejected": -256.9392395019531,
       "loss": 0.69,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.0038853283040225506,
-      "rewards/margins": 0.00776954460889101,
-      "rewards/rejected": -0.003884216072037816,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.005253526847809553,
+      "rewards/margins": 0.007615314330905676,
+      "rewards/rejected": -0.0023617881815880537,
       "step": 80
     },
     {
       "epoch": 0.18838304552590268,
-      "grad_norm": 0.4186108478199868,
+      "grad_norm": 0.41330526501243287,
       "learning_rate": 4.882681251368548e-07,
-      "logits/chosen": -0.9337117075920105,
-      "logits/rejected": -0.9007359743118286,
-      "logps/chosen": -257.81927490234375,
-      "logps/rejected": -270.7668151855469,
-      "loss": 0.6881,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.002065305132418871,
-      "rewards/margins": 0.010189032182097435,
-      "rewards/rejected": -0.012254336848855019,
+      "logits/chosen": -1.1750966310501099,
+      "logits/rejected": -1.1480153799057007,
+      "logps/chosen": -260.3459777832031,
+      "logps/rejected": -273.8209533691406,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.0038133221678435802,
+      "rewards/margins": 0.01101664174348116,
+      "rewards/rejected": -0.014829963445663452,
       "step": 90
     },
     {
       "epoch": 0.20931449502878074,
-      "grad_norm": 0.4458182507297675,
+      "grad_norm": 0.44625759181936003,
       "learning_rate": 4.820919832540181e-07,
-      "logits/chosen": -0.9627386927604675,
-      "logits/rejected": -0.9394882917404175,
-      "logps/chosen": -312.60650634765625,
-      "logps/rejected": -312.58447265625,
-      "loss": 0.6857,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.01190097164362669,
-      "rewards/margins": 0.019792668521404266,
-      "rewards/rejected": -0.03169363737106323,
+      "logits/chosen": -1.1883974075317383,
+      "logits/rejected": -1.1886602640151978,
+      "logps/chosen": -313.79180908203125,
+      "logps/rejected": -314.4971923828125,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.011269541457295418,
+      "rewards/margins": 0.02105005457997322,
+      "rewards/rejected": -0.03231959417462349,
       "step": 100
     },
     {
       "epoch": 0.2302459445316588,
-      "grad_norm": 0.7401018046473335,
+      "grad_norm": 0.8207302750561819,
       "learning_rate": 4.7467175306295647e-07,
-      "logits/chosen": -0.9153742790222168,
-      "logits/rejected": -0.8901848793029785,
-      "logps/chosen": -313.6246032714844,
-      "logps/rejected": -317.1287536621094,
-      "loss": 0.6836,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.023393774405121803,
-      "rewards/margins": 0.021103577688336372,
-      "rewards/rejected": -0.04449734836816788,
+      "logits/chosen": -1.151836633682251,
+      "logits/rejected": -1.1443371772766113,
+      "logps/chosen": -314.98638916015625,
+      "logps/rejected": -321.3186950683594,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.024794159457087517,
+      "rewards/margins": 0.023312237113714218,
+      "rewards/rejected": -0.048106398433446884,
       "step": 110
     },
     {
       "epoch": 0.25117739403453687,
-      "grad_norm": 0.8221557536673391,
+      "grad_norm": 0.8299428656973908,
       "learning_rate": 4.6604720940421207e-07,
-      "logits/chosen": -0.8409837484359741,
-      "logits/rejected": -0.8502853512763977,
-      "logps/chosen": -308.5286560058594,
-      "logps/rejected": -337.5436706542969,
-      "loss": 0.6781,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.039570122957229614,
-      "rewards/margins": 0.03355463594198227,
-      "rewards/rejected": -0.07312475144863129,
+      "logits/chosen": -1.078203797340393,
+      "logits/rejected": -1.097646951675415,
+      "logps/chosen": -310.5390625,
+      "logps/rejected": -342.7639465332031,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.04077012464404106,
+      "rewards/margins": 0.0370570607483387,
+      "rewards/rejected": -0.07782718539237976,
       "step": 120
     },
     {
       "epoch": 0.272108843537415,
-      "grad_norm": 1.1482922478086675,
+      "grad_norm": 1.9181550316420195,
       "learning_rate": 4.5626458262912735e-07,
-      "logits/chosen": -0.8868376612663269,
-      "logits/rejected": -0.8650667071342468,
-      "logps/chosen": -354.9898986816406,
-      "logps/rejected": -386.808349609375,
-      "loss": 0.6707,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.07423487305641174,
-      "rewards/margins": 0.040343888103961945,
-      "rewards/rejected": -0.11457876116037369,
+      "logits/chosen": -1.1113637685775757,
+      "logits/rejected": -1.1018600463867188,
+      "logps/chosen": -350.82489013671875,
+      "logps/rejected": -391.458984375,
+      "loss": 0.6687,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.06851955503225327,
+      "rewards/margins": 0.049427423626184464,
+      "rewards/rejected": -0.11794696748256683,
       "step": 130
     },
     {
       "epoch": 0.29304029304029305,
-      "grad_norm": 1.8202870864883658,
+      "grad_norm": 1.5901268072005463,
       "learning_rate": 4.453763107901675e-07,
-      "logits/chosen": -0.9577043652534485,
-      "logits/rejected": -0.9177180528640747,
-      "logps/chosen": -401.80108642578125,
-      "logps/rejected": -426.27777099609375,
-      "loss": 0.6643,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.08050010353326797,
-      "rewards/margins": 0.07808584719896317,
-      "rewards/rejected": -0.15858595073223114,
+      "logits/chosen": -1.1554381847381592,
+      "logits/rejected": -1.1351741552352905,
+      "logps/chosen": -387.03741455078125,
+      "logps/rejected": -405.8630676269531,
+      "loss": 0.6637,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.06498464196920395,
+      "rewards/margins": 0.07188350707292557,
+      "rewards/rejected": -0.1368681639432907,
       "step": 140
     },
     {
       "epoch": 0.3139717425431711,
-      "grad_norm": 1.8127314418328795,
+      "grad_norm": 1.5559930126192802,
       "learning_rate": 4.3344075855595097e-07,
-      "logits/chosen": -0.9365851283073425,
-      "logits/rejected": -0.9067959785461426,
-      "logps/chosen": -408.94659423828125,
-      "logps/rejected": -440.8140563964844,
-      "loss": 0.6619,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.12829507887363434,
-      "rewards/margins": 0.06538741290569305,
-      "rewards/rejected": -0.1936824768781662,
+      "logits/chosen": -1.1165324449539185,
+      "logits/rejected": -1.1060618162155151,
+      "logps/chosen": -384.8150329589844,
+      "logps/rejected": -410.9349060058594,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.10324742645025253,
+      "rewards/margins": 0.0590556375682354,
+      "rewards/rejected": -0.16230307519435883,
       "step": 150
     },
     {
       "epoch": 0.3349031920460492,
-      "grad_norm": 1.8472665134009132,
+      "grad_norm": 2.8840273514181995,
       "learning_rate": 4.2052190435769554e-07,
-      "logits/chosen": -0.9482985734939575,
-      "logits/rejected": -0.9125078320503235,
-      "logps/chosen": -465.48223876953125,
-      "logps/rejected": -529.677001953125,
-      "loss": 0.6572,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.18543408811092377,
-      "rewards/margins": 0.09176287055015564,
-      "rewards/rejected": -0.2771969437599182,
+      "logits/chosen": -1.121591329574585,
+      "logits/rejected": -1.1208221912384033,
+      "logps/chosen": -390.0221252441406,
+      "logps/rejected": -451.4759826660156,
+      "loss": 0.6586,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.10985617339611053,
+      "rewards/margins": 0.08711585402488708,
+      "rewards/rejected": -0.1969720423221588,
       "step": 160
     },
     {
       "epoch": 0.35583464154892724,
-      "grad_norm": 2.8194220987690435,
+      "grad_norm": 2.316912006659467,
       "learning_rate": 4.0668899744407567e-07,
-      "logits/chosen": -0.8278239965438843,
-      "logits/rejected": -0.8063944578170776,
-      "logps/chosen": -486.142578125,
-      "logps/rejected": -554.540283203125,
-      "loss": 0.6541,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.22446854412555695,
-      "rewards/margins": 0.09519634395837784,
-      "rewards/rejected": -0.3196648955345154,
+      "logits/chosen": -1.0214414596557617,
+      "logits/rejected": -1.0186623334884644,
+      "logps/chosen": -408.4321594238281,
+      "logps/rejected": -456.7073669433594,
+      "loss": 0.6582,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1461375653743744,
+      "rewards/margins": 0.07394389808177948,
+      "rewards/rejected": -0.22008147835731506,
       "step": 170
     },
     {
       "epoch": 0.37676609105180536,
-      "grad_norm": 2.97208521212178,
+      "grad_norm": 2.4425880971749514,
       "learning_rate": 3.920161866827889e-07,
-      "logits/chosen": -0.8902044296264648,
-      "logits/rejected": -0.8527171015739441,
-      "logps/chosen": -530.2462158203125,
-      "logps/rejected": -592.8751831054688,
-      "loss": 0.6406,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.2585751712322235,
-      "rewards/margins": 0.087519571185112,
-      "rewards/rejected": -0.3460947573184967,
+      "logits/chosen": -1.0814361572265625,
+      "logits/rejected": -1.060704231262207,
+      "logps/chosen": -425.4701232910156,
+      "logps/rejected": -474.9969787597656,
+      "loss": 0.6474,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.15264029800891876,
+      "rewards/margins": 0.07376497238874435,
+      "rewards/rejected": -0.22640526294708252,
       "step": 180
     },
     {
       "epoch": 0.3976975405546834,
-      "grad_norm": 2.702362177917078,
+      "grad_norm": 3.0318995366276247,
       "learning_rate": 3.765821230985757e-07,
-      "logits/chosen": -0.8619738817214966,
-      "logits/rejected": -0.8327099084854126,
-      "logps/chosen": -513.2433471679688,
-      "logps/rejected": -626.6792602539062,
-      "loss": 0.6462,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.2469058483839035,
-      "rewards/margins": 0.1172383576631546,
-      "rewards/rejected": -0.3641442060470581,
+      "logits/chosen": -1.0653070211410522,
+      "logits/rejected": -1.0440646409988403,
+      "logps/chosen": -428.57293701171875,
+      "logps/rejected": -503.21124267578125,
+      "loss": 0.6487,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.1615397036075592,
+      "rewards/margins": 0.07858813554048538,
+      "rewards/rejected": -0.24012784659862518,
       "step": 190
     },
     {
       "epoch": 0.4186289900575615,
-      "grad_norm": 3.8087843322902124,
+      "grad_norm": 4.194244230703589,
       "learning_rate": 3.604695382782159e-07,
-      "logits/chosen": -0.7917501926422119,
-      "logits/rejected": -0.8016451001167297,
-      "logps/chosen": -555.9425659179688,
-      "logps/rejected": -720.2203979492188,
-      "loss": 0.6402,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.3054454028606415,
-      "rewards/margins": 0.14363829791545868,
-      "rewards/rejected": -0.44908371567726135,
+      "logits/chosen": -1.0129735469818115,
+      "logits/rejected": -1.028363585472107,
+      "logps/chosen": -441.21502685546875,
+      "logps/rejected": -554.3182373046875,
+      "loss": 0.6476,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.18880845606327057,
+      "rewards/margins": 0.09274501353502274,
+      "rewards/rejected": -0.2815534770488739,
       "step": 200
     },
     {
       "epoch": 0.43956043956043955,
-      "grad_norm": 3.9347188229731374,
+      "grad_norm": 2.799782012874293,
       "learning_rate": 3.4376480090239047e-07,
-      "logits/chosen": -0.8363162279129028,
-      "logits/rejected": -0.7596572637557983,
-      "logps/chosen": -671.146728515625,
-      "logps/rejected": -783.2349243164062,
-      "loss": 0.6461,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.36001214385032654,
-      "rewards/margins": 0.16495202481746674,
-      "rewards/rejected": -0.5249641537666321,
+      "logits/chosen": -1.0415253639221191,
+      "logits/rejected": -1.0015658140182495,
+      "logps/chosen": -512.2741088867188,
+      "logps/rejected": -587.1564331054688,
+      "loss": 0.6457,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.19915422797203064,
+      "rewards/margins": 0.12847939133644104,
+      "rewards/rejected": -0.3276336193084717,
       "step": 210
     },
     {
       "epoch": 0.4604918890633176,
-      "grad_norm": 3.4218343739474046,
+      "grad_norm": 3.38423659902295,
       "learning_rate": 3.265574537815398e-07,
-      "logits/chosen": -0.7804139256477356,
-      "logits/rejected": -0.7788348197937012,
-      "logps/chosen": -537.9508666992188,
-      "logps/rejected": -720.7735595703125,
-      "loss": 0.6365,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.3049758970737457,
-      "rewards/margins": 0.17417363822460175,
-      "rewards/rejected": -0.4791494309902191,
+      "logits/chosen": -0.9850971102714539,
+      "logits/rejected": -1.0010693073272705,
+      "logps/chosen": -478.1683654785156,
+      "logps/rejected": -623.3740234375,
+      "loss": 0.6426,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.24368831515312195,
+      "rewards/margins": 0.13728444278240204,
+      "rewards/rejected": -0.3809727728366852,
       "step": 220
     },
     {
       "epoch": 0.48142333856619574,
-      "grad_norm": 3.795962622718272,
+      "grad_norm": 2.9823989997956555,
       "learning_rate": 3.0893973387735683e-07,
-      "logits/chosen": -0.8238471150398254,
-      "logits/rejected": -0.7790335416793823,
-      "logps/chosen": -618.4681396484375,
-      "logps/rejected": -830.87109375,
-      "loss": 0.6268,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.3598284125328064,
-      "rewards/margins": 0.23437073826789856,
-      "rewards/rejected": -0.5941991209983826,
+      "logits/chosen": -1.0038853883743286,
+      "logits/rejected": -0.9878044128417969,
+      "logps/chosen": -531.2225341796875,
+      "logps/rejected": -679.9888916015625,
+      "loss": 0.6402,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.2716134190559387,
+      "rewards/margins": 0.1706905961036682,
+      "rewards/rejected": -0.44230398535728455,
       "step": 230
     },
     {
       "epoch": 0.5023547880690737,
-      "grad_norm": 3.5058346085646144,
+      "grad_norm": 3.2499709469478084,
       "learning_rate": 2.910060778827554e-07,
-      "logits/chosen": -0.8182986378669739,
-      "logits/rejected": -0.7530924081802368,
-      "logps/chosen": -669.210205078125,
-      "logps/rejected": -834.8556518554688,
-      "loss": 0.6181,
+      "logits/chosen": -0.9987156987190247,
+      "logits/rejected": -0.9530145525932312,
+      "logps/chosen": -573.1682739257812,
+      "logps/rejected": -688.3665161132812,
+      "loss": 0.6257,
       "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.3839070200920105,
-      "rewards/margins": 0.1838807910680771,
-      "rewards/rejected": -0.567787766456604,
+      "rewards/chosen": -0.2874709367752075,
+      "rewards/margins": 0.1330692023038864,
+      "rewards/rejected": -0.4205401539802551,
       "step": 240
     },
     {
       "epoch": 0.5232862375719518,
-      "grad_norm": 3.784803079890303,
+      "grad_norm": 3.599189967926549,
       "learning_rate": 2.7285261601056697e-07,
-      "logits/chosen": -0.803874671459198,
-      "logits/rejected": -0.7223009467124939,
-      "logps/chosen": -647.1034545898438,
-      "logps/rejected": -853.0436401367188,
-      "loss": 0.6268,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.3654927909374237,
-      "rewards/margins": 0.24678358435630798,
-      "rewards/rejected": -0.6122764348983765,
+      "logits/chosen": -0.9661976099014282,
+      "logits/rejected": -0.922343909740448,
+      "logps/chosen": -613.0869140625,
+      "logps/rejected": -763.728515625,
+      "loss": 0.6364,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.33115440607070923,
+      "rewards/margins": 0.19019444286823273,
+      "rewards/rejected": -0.5213488340377808,
       "step": 250
     },
     {
       "epoch": 0.54421768707483,
-      "grad_norm": 5.477740532160938,
+      "grad_norm": 3.8677130438734486,
       "learning_rate": 2.5457665670441937e-07,
-      "logits/chosen": -0.8027833700180054,
-      "logits/rejected": -0.7885298728942871,
-      "logps/chosen": -674.1099853515625,
-      "logps/rejected": -937.3220825195312,
-      "loss": 0.6138,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.4139021039009094,
-      "rewards/margins": 0.27916184067726135,
-      "rewards/rejected": -0.6930639147758484,
+      "logits/chosen": -0.9890602231025696,
+      "logits/rejected": -0.987634003162384,
+      "logps/chosen": -588.9368896484375,
+      "logps/rejected": -765.2493286132812,
+      "loss": 0.6281,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.32784414291381836,
+      "rewards/margins": 0.19221974909305573,
+      "rewards/rejected": -0.5200638175010681,
       "step": 260
     },
     {
       "epoch": 0.565149136577708,
-      "grad_norm": 3.3580640957163905,
+      "grad_norm": 3.5281344720274674,
       "learning_rate": 2.3627616503391812e-07,
-      "logits/chosen": -0.7519720196723938,
-      "logits/rejected": -0.7142844796180725,
-      "logps/chosen": -751.9912719726562,
-      "logps/rejected": -1058.1048583984375,
-      "loss": 0.615,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.45110592246055603,
-      "rewards/margins": 0.3373827338218689,
-      "rewards/rejected": -0.7884886264801025,
+      "logits/chosen": -0.9270151853561401,
+      "logits/rejected": -0.9179307222366333,
+      "logps/chosen": -658.6036376953125,
+      "logps/rejected": -826.82666015625,
+      "loss": 0.6325,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.35761111974716187,
+      "rewards/margins": 0.19844211637973785,
+      "rewards/rejected": -0.5560532808303833,
       "step": 270
     },
     {
       "epoch": 0.5860805860805861,
-      "grad_norm": 5.5177213152440565,
+      "grad_norm": 3.407580559063376,
       "learning_rate": 2.1804923757009882e-07,
-      "logits/chosen": -0.6814185976982117,
-      "logits/rejected": -0.6468263864517212,
-      "logps/chosen": -705.2122192382812,
-      "logps/rejected": -920.8411865234375,
-      "loss": 0.6228,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.44767504930496216,
-      "rewards/margins": 0.24160179495811462,
-      "rewards/rejected": -0.6892768144607544,
+      "logits/chosen": -0.8880785703659058,
+      "logits/rejected": -0.8848760724067688,
+      "logps/chosen": -611.5701293945312,
+      "logps/rejected": -751.0750732421875,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.3527330756187439,
+      "rewards/margins": 0.16469557583332062,
+      "rewards/rejected": -0.5174286961555481,
       "step": 280
     },
     {
       "epoch": 0.6070120355834642,
-      "grad_norm": 4.551630518528997,
+      "grad_norm": 4.11114092580769,
       "learning_rate": 1.9999357655598891e-07,
-      "logits/chosen": -0.7232012152671814,
-      "logits/rejected": -0.6624692678451538,
-      "logps/chosen": -756.8780517578125,
-      "logps/rejected": -1010.7252197265625,
-      "loss": 0.6241,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.5086268782615662,
-      "rewards/margins": 0.2510340213775635,
-      "rewards/rejected": -0.7596608996391296,
+      "logits/chosen": -0.9366544485092163,
+      "logits/rejected": -0.8976960182189941,
+      "logps/chosen": -661.718505859375,
+      "logps/rejected": -826.75146484375,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.4124836027622223,
+      "rewards/margins": 0.16148509085178375,
+      "rewards/rejected": -0.5739686489105225,
       "step": 290
     },
     {
       "epoch": 0.6279434850863422,
-      "grad_norm": 5.147689539765831,
+      "grad_norm": 4.874958720462189,
       "learning_rate": 1.8220596619089573e-07,
-      "logits/chosen": -0.7237067818641663,
-      "logits/rejected": -0.6642863154411316,
-      "logps/chosen": -871.6720581054688,
-      "logps/rejected": -1069.462158203125,
-      "loss": 0.6007,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.542992115020752,
-      "rewards/margins": 0.24627108871936798,
-      "rewards/rejected": -0.7892631888389587,
+      "logits/chosen": -0.9249475598335266,
+      "logits/rejected": -0.8965902328491211,
+      "logps/chosen": -754.08154296875,
+      "logps/rejected": -854.07421875,
+      "loss": 0.6167,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4245515763759613,
+      "rewards/margins": 0.14848145842552185,
+      "rewards/rejected": -0.5730329751968384,
       "step": 300
     },
     {
       "epoch": 0.6488749345892203,
-      "grad_norm": 5.107775722831973,
+      "grad_norm": 5.165875801652537,
       "learning_rate": 1.647817538357072e-07,
-      "logits/chosen": -0.7091413736343384,
-      "logits/rejected": -0.6302168965339661,
-      "logps/chosen": -882.1735229492188,
-      "logps/rejected": -1218.36962890625,
-      "loss": 0.605,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.5749861598014832,
-      "rewards/margins": 0.3909863531589508,
-      "rewards/rejected": -0.9659725427627563,
+      "logits/chosen": -0.9006611704826355,
+      "logits/rejected": -0.8587985038757324,
+      "logps/chosen": -693.7996826171875,
+      "logps/rejected": -908.5036010742188,
+      "loss": 0.6186,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.3858186602592468,
+      "rewards/margins": 0.2699519693851471,
+      "rewards/rejected": -0.6557706594467163,
       "step": 310
     },
     {
       "epoch": 0.6698063840920984,
-      "grad_norm": 4.98218570447988,
+      "grad_norm": 3.598858462845785,
       "learning_rate": 1.478143389201113e-07,
-      "logits/chosen": -0.7340850830078125,
-      "logits/rejected": -0.6477845907211304,
-      "logps/chosen": -795.6258544921875,
-      "logps/rejected": -1128.2310791015625,
-      "loss": 0.5986,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.5385175943374634,
-      "rewards/margins": 0.3641910254955292,
-      "rewards/rejected": -0.9027086496353149,
+      "logits/chosen": -0.9343263506889343,
+      "logits/rejected": -0.8688840866088867,
+      "logps/chosen": -632.761474609375,
+      "logps/rejected": -885.478515625,
+      "loss": 0.6128,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3739064931869507,
+      "rewards/margins": 0.2838885486125946,
+      "rewards/rejected": -0.6577950716018677,
       "step": 320
     },
     {
       "epoch": 0.6907378335949764,
-      "grad_norm": 7.2646740452584195,
+      "grad_norm": 5.102953052593086,
       "learning_rate": 1.3139467229135998e-07,
-      "logits/chosen": -0.7201080322265625,
-      "logits/rejected": -0.6724112629890442,
-      "logps/chosen": -879.5479736328125,
-      "logps/rejected": -1196.298583984375,
-      "loss": 0.6091,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.6024604439735413,
-      "rewards/margins": 0.32484909892082214,
-      "rewards/rejected": -0.927309513092041,
+      "logits/chosen": -0.908793568611145,
+      "logits/rejected": -0.8711221814155579,
+      "logps/chosen": -728.1497802734375,
+      "logps/rejected": -968.7476806640625,
+      "loss": 0.6215,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4502260088920593,
+      "rewards/margins": 0.24815288186073303,
+      "rewards/rejected": -0.6983789205551147,
       "step": 330
     },
     {
       "epoch": 0.7116692830978545,
-      "grad_norm": 7.541765265306689,
+      "grad_norm": 5.671673157970287,
       "learning_rate": 1.1561076868822755e-07,
-      "logits/chosen": -0.613195538520813,
-      "logits/rejected": -0.5763617753982544,
-      "logps/chosen": -933.0261840820312,
-      "logps/rejected": -1260.025634765625,
-      "loss": 0.5923,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.6348632574081421,
-      "rewards/margins": 0.35707950592041016,
-      "rewards/rejected": -0.9919427633285522,
+      "logits/chosen": -0.8156379461288452,
+      "logits/rejected": -0.8025031089782715,
+      "logps/chosen": -742.4412841796875,
+      "logps/rejected": -978.5466918945312,
+      "loss": 0.6096,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.44500136375427246,
+      "rewards/margins": 0.2643851637840271,
+      "rewards/rejected": -0.7093865275382996,
       "step": 340
     },
     {
       "epoch": 0.7326007326007326,
-      "grad_norm": 5.982906996429055,
+      "grad_norm": 4.788572684554985,
       "learning_rate": 1.0054723495346482e-07,
-      "logits/chosen": -0.6862212419509888,
-      "logits/rejected": -0.6319141387939453,
-      "logps/chosen": -905.0227661132812,
-      "logps/rejected": -1189.1097412109375,
-      "loss": 0.5771,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.6419159173965454,
-      "rewards/margins": 0.3145480751991272,
-      "rewards/rejected": -0.9564639925956726,
+      "logits/chosen": -0.8821672201156616,
+      "logits/rejected": -0.8667781949043274,
+      "logps/chosen": -744.6392822265625,
+      "logps/rejected": -950.1046142578125,
+      "loss": 0.6065,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.4801097810268402,
+      "rewards/margins": 0.23577599227428436,
+      "rewards/rejected": -0.7158857583999634,
       "step": 350
     },
     {
       "epoch": 0.7535321821036107,
-      "grad_norm": 5.935343010634375,
+      "grad_norm": 4.756088898998946,
       "learning_rate": 8.628481651367875e-08,
-      "logits/chosen": -0.6817030310630798,
-      "logits/rejected": -0.6138208508491516,
-      "logps/chosen": -1023.2503662109375,
-      "logps/rejected": -1340.0045166015625,
-      "loss": 0.6097,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.7090341448783875,
-      "rewards/margins": 0.35953468084335327,
-      "rewards/rejected": -1.0685687065124512,
+      "logits/chosen": -0.8794587850570679,
+      "logits/rejected": -0.8525463342666626,
+      "logps/chosen": -842.9227294921875,
+      "logps/rejected": -1010.73828125,
+      "loss": 0.6269,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.528265655040741,
+      "rewards/margins": 0.20947960019111633,
+      "rewards/rejected": -0.7377452254295349,
       "step": 360
     },
     {
       "epoch": 0.7744636316064888,
-      "grad_norm": 4.786610680497262,
+      "grad_norm": 4.287691995792507,
       "learning_rate": 7.289996455765748e-08,
-      "logits/chosen": -0.7248352766036987,
-      "logits/rejected": -0.6292211413383484,
-      "logps/chosen": -842.8912353515625,
-      "logps/rejected": -1210.052490234375,
-      "loss": 0.597,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.5791453123092651,
-      "rewards/margins": 0.39849239587783813,
-      "rewards/rejected": -0.977637767791748,
+      "logits/chosen": -0.9290102124214172,
+      "logits/rejected": -0.8693262338638306,
+      "logps/chosen": -681.6297607421875,
+      "logps/rejected": -900.4710083007812,
+      "loss": 0.6112,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.41641873121261597,
+      "rewards/margins": 0.2501312494277954,
+      "rewards/rejected": -0.6665499806404114,
       "step": 370
     },
     {
       "epoch": 0.7953950811093669,
-      "grad_norm": 5.360927626743976,
+      "grad_norm": 3.351788727441381,
       "learning_rate": 6.046442623320145e-08,
-      "logits/chosen": -0.5932396650314331,
-      "logits/rejected": -0.5533714294433594,
-      "logps/chosen": -919.9244995117188,
-      "logps/rejected": -1385.126953125,
-      "loss": 0.5988,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.6735996007919312,
-      "rewards/margins": 0.44867175817489624,
-      "rewards/rejected": -1.1222712993621826,
+      "logits/chosen": -0.8099156618118286,
+      "logits/rejected": -0.7845919728279114,
+      "logps/chosen": -751.2888793945312,
+      "logps/rejected": -1116.771484375,
+      "loss": 0.6152,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.5037558674812317,
+      "rewards/margins": 0.3494088351726532,
+      "rewards/rejected": -0.8531646728515625,
       "step": 380
     },
     {
       "epoch": 0.8163265306122449,
-      "grad_norm": 7.044779963713088,
+      "grad_norm": 4.2822185775858355,
       "learning_rate": 4.904486005914027e-08,
-      "logits/chosen": -0.713057279586792,
-      "logits/rejected": -0.6076905131340027,
-      "logps/chosen": -975.0090942382812,
-      "logps/rejected": -1386.0218505859375,
-      "loss": 0.5884,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.6378625631332397,
-      "rewards/margins": 0.4438837468624115,
-      "rewards/rejected": -1.0817463397979736,
+      "logits/chosen": -0.8926030993461609,
+      "logits/rejected": -0.8195274472236633,
+      "logps/chosen": -821.78125,
+      "logps/rejected": -1080.7801513671875,
+      "loss": 0.6109,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.4835873544216156,
+      "rewards/margins": 0.2913810908794403,
+      "rewards/rejected": -0.7749683856964111,
       "step": 390
     },
     {
       "epoch": 0.837257980115123,
-      "grad_norm": 6.339569576065322,
+      "grad_norm": 3.832733560631146,
       "learning_rate": 3.8702478614051345e-08,
-      "logits/chosen": -0.6517136693000793,
-      "logits/rejected": -0.589192807674408,
-      "logps/chosen": -841.0965576171875,
-      "logps/rejected": -1160.896484375,
-      "loss": 0.5985,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.577842652797699,
-      "rewards/margins": 0.3430059850215912,
-      "rewards/rejected": -0.9208486676216125,
+      "logits/chosen": -0.8491037487983704,
+      "logits/rejected": -0.8279684782028198,
+      "logps/chosen": -704.4039916992188,
+      "logps/rejected": -935.1871337890625,
+      "loss": 0.6164,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.44019538164138794,
+      "rewards/margins": 0.25378626585006714,
+      "rewards/rejected": -0.6939816474914551,
       "step": 400
     },
     {
       "epoch": 0.858189429618001,
-      "grad_norm": 5.268821551699033,
+      "grad_norm": 3.767221913490684,
       "learning_rate": 2.9492720416985e-08,
-      "logits/chosen": -0.7182366847991943,
-      "logits/rejected": -0.6271320581436157,
-      "logps/chosen": -902.9696044921875,
-      "logps/rejected": -1317.0338134765625,
-      "loss": 0.5968,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.5933186411857605,
-      "rewards/margins": 0.4583154320716858,
-      "rewards/rejected": -1.0516341924667358,
+      "logits/chosen": -0.8924685716629028,
+      "logits/rejected": -0.858057975769043,
+      "logps/chosen": -775.3809814453125,
+      "logps/rejected": -1018.1121215820312,
+      "loss": 0.6163,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.46508699655532837,
+      "rewards/margins": 0.28694623708724976,
+      "rewards/rejected": -0.7520332932472229,
       "step": 410
     },
     {
       "epoch": 0.8791208791208791,
-      "grad_norm": 6.427134267894669,
+      "grad_norm": 5.174199687247601,
       "learning_rate": 2.1464952759020856e-08,
-      "logits/chosen": -0.5536683797836304,
-      "logits/rejected": -0.5348982810974121,
-      "logps/chosen": -916.4422607421875,
-      "logps/rejected": -1366.26171875,
-      "loss": 0.5804,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.6526989936828613,
-      "rewards/margins": 0.44348520040512085,
-      "rewards/rejected": -1.0961841344833374,
+      "logits/chosen": -0.7606777548789978,
+      "logits/rejected": -0.7737725377082825,
+      "logps/chosen": -774.3548583984375,
+      "logps/rejected": -1066.3587646484375,
+      "loss": 0.6037,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5085434317588806,
+      "rewards/margins": 0.28706783056259155,
+      "rewards/rejected": -0.7956112623214722,
       "step": 420
     },
     {
       "epoch": 0.9000523286237572,
-      "grad_norm": 5.223662175792827,
+      "grad_norm": 3.6763407260644145,
       "learning_rate": 1.4662207078575684e-08,
-      "logits/chosen": -0.6187500357627869,
-      "logits/rejected": -0.5157658457756042,
-      "logps/chosen": -987.8230590820312,
-      "logps/rejected": -1380.6385498046875,
-      "loss": 0.5858,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.6942586898803711,
-      "rewards/margins": 0.41675519943237305,
-      "rewards/rejected": -1.1110140085220337,
+      "logits/chosen": -0.8245135545730591,
+      "logits/rejected": -0.7580205202102661,
+      "logps/chosen": -801.9810791015625,
+      "logps/rejected": -1054.1705322265625,
+      "loss": 0.6119,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5072224736213684,
+      "rewards/margins": 0.2761218249797821,
+      "rewards/rejected": -0.7833442091941833,
       "step": 430
     },
     {
       "epoch": 0.9209837781266352,
-      "grad_norm": 6.9904169822044855,
+      "grad_norm": 3.9565073368902963,
       "learning_rate": 9.12094829893642e-09,
-      "logits/chosen": -0.6982117891311646,
-      "logits/rejected": -0.6187722682952881,
-      "logps/chosen": -875.1052856445312,
-      "logps/rejected": -1151.622314453125,
-      "loss": 0.5794,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.6084715127944946,
-      "rewards/margins": 0.3183104991912842,
-      "rewards/rejected": -0.9267820119857788,
+      "logits/chosen": -0.8855506777763367,
+      "logits/rejected": -0.8454933166503906,
+      "logps/chosen": -754.6903076171875,
+      "logps/rejected": -885.1017456054688,
+      "loss": 0.6043,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.4870569705963135,
+      "rewards/margins": 0.17229454219341278,
+      "rewards/rejected": -0.6593514680862427,
       "step": 440
     },
     {
       "epoch": 0.9419152276295133,
-      "grad_norm": 5.431378633125538,
+      "grad_norm": 3.701479740364913,
       "learning_rate": 4.8708793644441086e-09,
-      "logits/chosen": -0.5444275140762329,
-      "logits/rejected": -0.48638853430747986,
-      "logps/chosen": -933.2413940429688,
-      "logps/rejected": -1319.6456298828125,
-      "loss": 0.5906,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.6616878509521484,
-      "rewards/margins": 0.39823150634765625,
-      "rewards/rejected": -1.0599192380905151,
+      "logits/chosen": -0.7477255463600159,
+      "logits/rejected": -0.7270427346229553,
+      "logps/chosen": -780.8983154296875,
+      "logps/rejected": -1019.4451293945312,
+      "loss": 0.6124,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5077296495437622,
+      "rewards/margins": 0.251300185918808,
+      "rewards/rejected": -0.7590298056602478,
       "step": 450
     },
     {
       "epoch": 0.9628466771323915,
-      "grad_norm": 6.561608804085118,
+      "grad_norm": 5.067825971160372,
       "learning_rate": 1.9347820230782295e-09,
-      "logits/chosen": -0.6087552905082703,
-      "logits/rejected": -0.5210331678390503,
-      "logps/chosen": -938.4767456054688,
-      "logps/rejected": -1216.3330078125,
-      "loss": 0.6007,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.6643339395523071,
-      "rewards/margins": 0.32233306765556335,
-      "rewards/rejected": -0.9866669774055481,
+      "logits/chosen": -0.7995008230209351,
+      "logits/rejected": -0.7618998289108276,
+      "logps/chosen": -767.2562866210938,
+      "logps/rejected": -937.3308715820312,
+      "loss": 0.6147,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.49197807908058167,
+      "rewards/margins": 0.2139851152896881,
+      "rewards/rejected": -0.705963134765625,
       "step": 460
     },
     {
       "epoch": 0.9837781266352695,
-      "grad_norm": 7.959100177075258,
+      "grad_norm": 5.987105760607766,
       "learning_rate": 3.2839470889836627e-10,
-      "logits/chosen": -0.657787024974823,
-      "logits/rejected": -0.610327422618866,
-      "logps/chosen": -939.5802001953125,
-      "logps/rejected": -1180.6595458984375,
-      "loss": 0.5855,
+      "logits/chosen": -0.8542754054069519,
+      "logits/rejected": -0.8349153399467468,
+      "logps/chosen": -772.111328125,
+      "logps/rejected": -954.55517578125,
+      "loss": 0.6047,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.6418840885162354,
-      "rewards/margins": 0.2594161033630371,
-      "rewards/rejected": -0.9013002514839172,
+      "rewards/chosen": -0.47330260276794434,
+      "rewards/margins": 0.19964265823364258,
+      "rewards/rejected": -0.6729452610015869,
       "step": 470
     },
     {
       "epoch": 0.9984301412872841,
-      "eval_logits/chosen": -0.6367819905281067,
-      "eval_logits/rejected": -0.5771758556365967,
-      "eval_logps/chosen": -907.722900390625,
-      "eval_logps/rejected": -1395.880859375,
-      "eval_loss": 0.5986923575401306,
-      "eval_rewards/accuracies": 0.70703125,
-      "eval_rewards/chosen": -0.6320347189903259,
-      "eval_rewards/margins": 0.4867088794708252,
-      "eval_rewards/rejected": -1.118743658065796,
-      "eval_runtime": 222.6786,
-      "eval_samples_per_second": 8.982,
-      "eval_steps_per_second": 0.144,
+      "eval_logits/chosen": -0.8622139692306519,
+      "eval_logits/rejected": -0.8340398669242859,
+      "eval_logps/chosen": -740.609375,
+      "eval_logps/rejected": -1076.8990478515625,
+      "eval_loss": 0.6172269582748413,
+      "eval_rewards/accuracies": 0.75,
+      "eval_rewards/chosen": -0.46546465158462524,
+      "eval_rewards/margins": 0.33346831798553467,
+      "eval_rewards/rejected": -0.7989329099655151,
+      "eval_runtime": 191.0283,
+      "eval_samples_per_second": 10.47,
+      "eval_steps_per_second": 0.168,
       "step": 477
     },
     {
       "epoch": 0.9984301412872841,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.6339174956395691,
-      "train_runtime": 13552.9806,
-      "train_samples_per_second": 4.511,
+      "train_loss": 0.6437127205310877,
+      "train_runtime": 13588.3404,
+      "train_samples_per_second": 4.499,
       "train_steps_per_second": 0.035
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d60eab87c985a5c7650684bc1ed6bd80433e9b31291eda50488d9bc6ce7211a
+oid sha256:1c528bf9faa59f5082bdd7cb4aa549d795a0bd83e24252c64269557f86a809bf
 size 7672