Model save

README.md

@@ -16,15 +16,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6172
-- Rewards/chosen: -0.4655
-- Rewards/rejected: -0.7989
-- Rewards/accuracies: 0.75
-- Rewards/margins: 0.3335
-- Logps/rejected: -1076.8990
-- Logps/chosen: -740.6094
-- Logits/rejected: -0.8340
-- Logits/chosen: -0.8622
+- Loss: 0.6100
+- Rewards/chosen: -0.5090
+- Rewards/rejected: -0.8748
+- Rewards/accuracies: 0.7383
+- Rewards/margins: 0.3658
+- Logps/rejected: -1150.8049
+- Logps/chosen: -782.0501
+- Logits/rejected: -0.8126
+- Logits/chosen: -0.8429
 
 ## Model description
 
@@ -61,7 +61,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6047        | 0.9984 | 477  | 0.6172          | -0.4655        | -0.7989          | 0.75               | 0.3335          | -1076.8990     | -740.6094    | -0.8340         | -0.8622       |
+| 0.5979        | 0.9984 | 477  | 0.6100          | -0.5090        | -0.8748          | 0.7383             | 0.3658          | -1150.8049     | -782.0501    | -0.8126         | -0.8429       |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.6437127205310877,
-    "train_runtime": 13588.3404,
+    "train_loss": 0.6406088435175034,
+    "train_runtime": 13597.1074,
     "train_samples": 61134,
-    "train_samples_per_second": 4.499,
+    "train_samples_per_second": 4.496,
     "train_steps_per_second": 0.035
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/scratch/pawsey0935/tliu1/alignment-handbook/data/zephyr-8b-sft-1e/checkpoint-500",
+  "_name_or_path": "/scratch/pawsey0935/tliu1/alignment-handbook/data/zephyr-8b-sft-1e/checkpoint-600",
   "architectures": [
     "LlamaForCausalLM"
   ],

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f57f7d4d01531c960492b69349323e79d4f1ef8b1e4f17132ee8910ea6aa00aa
+oid sha256:a046a3ed6a61096279209df3346e40a93ae12453504031ed4d2f06339fc24c2d
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a378753606915dbc7f6fad865ae1ac46232454da1d3053dad2e0bf172229c27
+oid sha256:9eaf50bc9079622cac39855516a677fdee4da87dddecf5c835970eb9214a1b67
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa19495c5134915d35e7027ccafd563f0dce9a2650b1db3049af49b6069f63c7
+oid sha256:c22129b0846c57116cfa5ad8466f51565aca7b4862a26b36dd4a798fdcdeeefc
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac5f251487a2101a2dd9ce3b7e050ab2abd396451140fb470ac84dca406138de
+oid sha256:c9b0603f0e61bad2aeda15c3c2ebe7a71d1e072496deb012f99b0a359e301d3c
 size 1168138808

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.6437127205310877,
-    "train_runtime": 13588.3404,
+    "train_loss": 0.6406088435175034,
+    "train_runtime": 13597.1074,
     "train_samples": 61134,
-    "train_samples_per_second": 4.499,
+    "train_samples_per_second": 4.496,
     "train_steps_per_second": 0.035
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0020931449502878076,
-      "grad_norm": 0.3180850451895604,
+      "grad_norm": 0.32324101626998236,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -1.2097231149673462,
-      "logits/rejected": -1.2087113857269287,
-      "logps/chosen": -360.4364013671875,
-      "logps/rejected": -362.17816162109375,
+      "logits/chosen": -1.254675030708313,
+      "logits/rejected": -1.2519813776016235,
+      "logps/chosen": -357.39349365234375,
+      "logps/rejected": -360.9093322753906,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,732 +25,732 @@
     },
     {
       "epoch": 0.020931449502878074,
-      "grad_norm": 0.3091562668837711,
+      "grad_norm": 0.31303582401914404,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -1.1554023027420044,
-      "logits/rejected": -1.1447501182556152,
-      "logps/chosen": -302.14666748046875,
-      "logps/rejected": -278.4107360839844,
+      "logits/chosen": -1.1969424486160278,
+      "logits/rejected": -1.1853214502334595,
+      "logps/chosen": -299.4916687011719,
+      "logps/rejected": -277.1794128417969,
       "loss": 0.6931,
-      "rewards/accuracies": 0.4930555522441864,
-      "rewards/chosen": 7.456003459083149e-06,
-      "rewards/margins": 3.9713955629849806e-05,
-      "rewards/rejected": -3.225794353056699e-05,
+      "rewards/accuracies": 0.4236111044883728,
+      "rewards/chosen": 2.0686069547082298e-05,
+      "rewards/margins": 2.5644327251939103e-05,
+      "rewards/rejected": -4.958255431120051e-06,
       "step": 10
     },
     {
       "epoch": 0.04186289900575615,
-      "grad_norm": 0.29442577149940785,
+      "grad_norm": 0.2946808627363446,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -1.1197772026062012,
-      "logits/rejected": -1.1169277429580688,
-      "logps/chosen": -313.89813232421875,
-      "logps/rejected": -281.2487487792969,
+      "logits/chosen": -1.1608688831329346,
+      "logits/rejected": -1.1570008993148804,
+      "logps/chosen": -311.9692077636719,
+      "logps/rejected": -279.7148742675781,
       "loss": 0.6931,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 3.8420424971263856e-05,
-      "rewards/margins": -3.498773367027752e-05,
-      "rewards/rejected": 7.340816227952018e-05,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 5.6396173022221774e-05,
+      "rewards/margins": 1.9542889276635833e-05,
+      "rewards/rejected": 3.6853296478511766e-05,
       "step": 20
     },
     {
       "epoch": 0.06279434850863422,
-      "grad_norm": 0.2970425624012531,
+      "grad_norm": 0.29651950322714443,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -1.1332579851150513,
-      "logits/rejected": -1.1339633464813232,
-      "logps/chosen": -291.05224609375,
-      "logps/rejected": -250.421142578125,
+      "logits/chosen": -1.178495168685913,
+      "logits/rejected": -1.1777125597000122,
+      "logps/chosen": -289.05224609375,
+      "logps/rejected": -248.8190460205078,
       "loss": 0.6931,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 0.0005964103038422763,
-      "rewards/margins": 0.0001513070019427687,
-      "rewards/rejected": 0.0004451033310033381,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.0005276039009913802,
+      "rewards/margins": 0.0001321145100519061,
+      "rewards/rejected": 0.0003954893327318132,
       "step": 30
     },
     {
       "epoch": 0.0837257980115123,
-      "grad_norm": 0.29488203891244075,
+      "grad_norm": 0.2988220823405562,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -1.1262879371643066,
-      "logits/rejected": -1.118971347808838,
-      "logps/chosen": -259.94232177734375,
-      "logps/rejected": -261.27496337890625,
+      "logits/chosen": -1.1629207134246826,
+      "logits/rejected": -1.1547033786773682,
+      "logps/chosen": -257.3613586425781,
+      "logps/rejected": -259.5196228027344,
       "loss": 0.6929,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.0016647384036332369,
-      "rewards/margins": 0.000596812751609832,
-      "rewards/rejected": 0.001067925593815744,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0016067821998149157,
+      "rewards/margins": 0.0005107645411044359,
+      "rewards/rejected": 0.0010960176587104797,
       "step": 40
     },
     {
       "epoch": 0.10465724751439037,
-      "grad_norm": 0.3058449366923936,
+      "grad_norm": 0.3125736010449426,
       "learning_rate": 4.999731868769026e-07,
-      "logits/chosen": -1.1080738306045532,
-      "logits/rejected": -1.1030961275100708,
-      "logps/chosen": -287.21307373046875,
-      "logps/rejected": -272.30548095703125,
+      "logits/chosen": -1.1483451128005981,
+      "logits/rejected": -1.1426894664764404,
+      "logps/chosen": -285.2168273925781,
+      "logps/rejected": -270.5537414550781,
       "loss": 0.6925,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.0033532450906932354,
-      "rewards/margins": 0.0014802367659285665,
-      "rewards/rejected": 0.001873008324764669,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0029888246208429337,
+      "rewards/margins": 0.00123431789688766,
+      "rewards/rejected": 0.0017545067239552736,
       "step": 50
     },
     {
       "epoch": 0.12558869701726844,
-      "grad_norm": 0.32524143765136054,
+      "grad_norm": 0.32861986652173775,
       "learning_rate": 4.990353313429303e-07,
-      "logits/chosen": -1.1249278783798218,
-      "logits/rejected": -1.1207427978515625,
-      "logps/chosen": -252.5863494873047,
-      "logps/rejected": -243.4455108642578,
+      "logits/chosen": -1.1673046350479126,
+      "logits/rejected": -1.1617118120193481,
+      "logps/chosen": -250.5665283203125,
+      "logps/rejected": -242.58969116210938,
       "loss": 0.6919,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.006057645194232464,
-      "rewards/margins": 0.002920362865552306,
-      "rewards/rejected": 0.0031372823286801577,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.005732725840061903,
+      "rewards/margins": 0.0029540960676968098,
+      "rewards/rejected": 0.002778630005195737,
       "step": 60
     },
     {
       "epoch": 0.14652014652014653,
-      "grad_norm": 0.2965997309874686,
+      "grad_norm": 0.3018029007900129,
       "learning_rate": 4.967625656594781e-07,
-      "logits/chosen": -1.124455451965332,
-      "logits/rejected": -1.1197750568389893,
-      "logps/chosen": -292.89508056640625,
-      "logps/rejected": -281.67022705078125,
-      "loss": 0.6911,
+      "logits/chosen": -1.1635328531265259,
+      "logits/rejected": -1.1576188802719116,
+      "logps/chosen": -292.01702880859375,
+      "logps/rejected": -280.9066162109375,
+      "loss": 0.691,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.006117576267570257,
-      "rewards/margins": 0.004100095480680466,
-      "rewards/rejected": 0.0020174814853817225,
+      "rewards/chosen": 0.005355691071599722,
+      "rewards/margins": 0.0041871508583426476,
+      "rewards/rejected": 0.0011685403296723962,
       "step": 70
     },
     {
       "epoch": 0.1674515960230246,
-      "grad_norm": 0.3681249264262344,
+      "grad_norm": 0.3706523862968378,
       "learning_rate": 4.93167072587771e-07,
-      "logits/chosen": -1.1383342742919922,
-      "logits/rejected": -1.123975396156311,
-      "logps/chosen": -325.63470458984375,
-      "logps/rejected": -256.9392395019531,
-      "loss": 0.69,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.005253526847809553,
-      "rewards/margins": 0.007615314330905676,
-      "rewards/rejected": -0.0023617881815880537,
+      "logits/chosen": -1.1743751764297485,
+      "logits/rejected": -1.1609123945236206,
+      "logps/chosen": -324.23712158203125,
+      "logps/rejected": -257.1612854003906,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.0037769668269902468,
+      "rewards/margins": 0.007478375919163227,
+      "rewards/rejected": -0.0037014088593423367,
       "step": 80
     },
     {
       "epoch": 0.18838304552590268,
-      "grad_norm": 0.41330526501243287,
+      "grad_norm": 0.40916342963365604,
       "learning_rate": 4.882681251368548e-07,
-      "logits/chosen": -1.1750966310501099,
-      "logits/rejected": -1.1480153799057007,
-      "logps/chosen": -260.3459777832031,
-      "logps/rejected": -273.8209533691406,
+      "logits/chosen": -1.210192084312439,
+      "logits/rejected": -1.1824644804000854,
+      "logps/chosen": -260.08148193359375,
+      "logps/rejected": -273.74151611328125,
       "loss": 0.6878,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.0038133221678435802,
-      "rewards/margins": 0.01101664174348116,
-      "rewards/rejected": -0.014829963445663452,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.005586659070104361,
+      "rewards/margins": 0.010486402548849583,
+      "rewards/rejected": -0.016073061153292656,
       "step": 90
     },
     {
       "epoch": 0.20931449502878074,
-      "grad_norm": 0.44625759181936003,
+      "grad_norm": 0.4754293468181655,
       "learning_rate": 4.820919832540181e-07,
-      "logits/chosen": -1.1883974075317383,
-      "logits/rejected": -1.1886602640151978,
-      "logps/chosen": -313.79180908203125,
-      "logps/rejected": -314.4971923828125,
+      "logits/chosen": -1.2199891805648804,
+      "logits/rejected": -1.220046043395996,
+      "logps/chosen": -312.9206848144531,
+      "logps/rejected": -314.90960693359375,
       "loss": 0.6854,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.011269541457295418,
-      "rewards/margins": 0.02105005457997322,
-      "rewards/rejected": -0.03231959417462349,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.013386862352490425,
+      "rewards/margins": 0.020824002102017403,
+      "rewards/rejected": -0.03421086445450783,
       "step": 100
     },
     {
       "epoch": 0.2302459445316588,
-      "grad_norm": 0.8207302750561819,
+      "grad_norm": 0.8738829496526755,
       "learning_rate": 4.7467175306295647e-07,
-      "logits/chosen": -1.151836633682251,
-      "logits/rejected": -1.1443371772766113,
-      "logps/chosen": -314.98638916015625,
-      "logps/rejected": -321.3186950683594,
+      "logits/chosen": -1.1820493936538696,
+      "logits/rejected": -1.1728696823120117,
+      "logps/chosen": -311.4639587402344,
+      "logps/rejected": -316.75201416015625,
       "loss": 0.6831,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.024794159457087517,
-      "rewards/margins": 0.023312237113714218,
-      "rewards/rejected": -0.048106398433446884,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.023116732016205788,
+      "rewards/margins": 0.02223314717411995,
+      "rewards/rejected": -0.045349881052970886,
       "step": 110
     },
     {
       "epoch": 0.25117739403453687,
-      "grad_norm": 0.8299428656973908,
+      "grad_norm": 0.8961784473053371,
       "learning_rate": 4.6604720940421207e-07,
-      "logits/chosen": -1.078203797340393,
-      "logits/rejected": -1.097646951675415,
-      "logps/chosen": -310.5390625,
-      "logps/rejected": -342.7639465332031,
-      "loss": 0.6755,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.04077012464404106,
-      "rewards/margins": 0.0370570607483387,
-      "rewards/rejected": -0.07782718539237976,
+      "logits/chosen": -1.1100571155548096,
+      "logits/rejected": -1.1283613443374634,
+      "logps/chosen": -302.40948486328125,
+      "logps/rejected": -335.69512939453125,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.03481901437044144,
+      "rewards/margins": 0.0378703810274601,
+      "rewards/rejected": -0.07268939912319183,
       "step": 120
     },
     {
       "epoch": 0.272108843537415,
-      "grad_norm": 1.9181550316420195,
+      "grad_norm": 1.497566674498532,
       "learning_rate": 4.5626458262912735e-07,
-      "logits/chosen": -1.1113637685775757,
-      "logits/rejected": -1.1018600463867188,
-      "logps/chosen": -350.82489013671875,
-      "logps/rejected": -391.458984375,
-      "loss": 0.6687,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.06851955503225327,
-      "rewards/margins": 0.049427423626184464,
-      "rewards/rejected": -0.11794696748256683,
+      "logits/chosen": -1.14939284324646,
+      "logits/rejected": -1.142646074295044,
+      "logps/chosen": -352.44049072265625,
+      "logps/rejected": -388.8111267089844,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.07200166583061218,
+      "rewards/margins": 0.04512103646993637,
+      "rewards/rejected": -0.11712269484996796,
       "step": 130
     },
     {
       "epoch": 0.29304029304029305,
-      "grad_norm": 1.5901268072005463,
+      "grad_norm": 2.001200835384951,
       "learning_rate": 4.453763107901675e-07,
-      "logits/chosen": -1.1554381847381592,
-      "logits/rejected": -1.1351741552352905,
-      "logps/chosen": -387.03741455078125,
-      "logps/rejected": -405.8630676269531,
-      "loss": 0.6637,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.06498464196920395,
-      "rewards/margins": 0.07188350707292557,
-      "rewards/rejected": -0.1368681639432907,
+      "logits/chosen": -1.196212649345398,
+      "logits/rejected": -1.1793403625488281,
+      "logps/chosen": -385.7120056152344,
+      "logps/rejected": -407.84112548828125,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.06641264259815216,
+      "rewards/margins": 0.07408300787210464,
+      "rewards/rejected": -0.1404956430196762,
       "step": 140
     },
     {
       "epoch": 0.3139717425431711,
-      "grad_norm": 1.5559930126192802,
+      "grad_norm": 1.97796076277746,
       "learning_rate": 4.3344075855595097e-07,
-      "logits/chosen": -1.1165324449539185,
-      "logits/rejected": -1.1060618162155151,
-      "logps/chosen": -384.8150329589844,
-      "logps/rejected": -410.9349060058594,
-      "loss": 0.6638,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.10324742645025253,
-      "rewards/margins": 0.0590556375682354,
-      "rewards/rejected": -0.16230307519435883,
+      "logits/chosen": -1.15965735912323,
+      "logits/rejected": -1.1490386724472046,
+      "logps/chosen": -375.55120849609375,
+      "logps/rejected": -405.73077392578125,
+      "loss": 0.6608,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.09626610577106476,
+      "rewards/margins": 0.06254641711711884,
+      "rewards/rejected": -0.1588125228881836,
       "step": 150
     },
     {
       "epoch": 0.3349031920460492,
-      "grad_norm": 2.8840273514181995,
+      "grad_norm": 3.0008548992733766,
       "learning_rate": 4.2052190435769554e-07,
-      "logits/chosen": -1.121591329574585,
-      "logits/rejected": -1.1208221912384033,
-      "logps/chosen": -390.0221252441406,
-      "logps/rejected": -451.4759826660156,
-      "loss": 0.6586,
+      "logits/chosen": -1.1561930179595947,
+      "logits/rejected": -1.1568859815597534,
+      "logps/chosen": -409.51068115234375,
+      "logps/rejected": -472.51080322265625,
+      "loss": 0.6568,
       "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.10985617339611053,
-      "rewards/margins": 0.08711585402488708,
-      "rewards/rejected": -0.1969720423221588,
+      "rewards/chosen": -0.1316855102777481,
+      "rewards/margins": 0.08769272267818451,
+      "rewards/rejected": -0.21937823295593262,
       "step": 160
     },
     {
       "epoch": 0.35583464154892724,
-      "grad_norm": 2.316912006659467,
+      "grad_norm": 1.9634265810681608,
       "learning_rate": 4.0668899744407567e-07,
-      "logits/chosen": -1.0214414596557617,
-      "logits/rejected": -1.0186623334884644,
-      "logps/chosen": -408.4321594238281,
-      "logps/rejected": -456.7073669433594,
-      "loss": 0.6582,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.1461375653743744,
-      "rewards/margins": 0.07394389808177948,
-      "rewards/rejected": -0.22008147835731506,
+      "logits/chosen": -1.048796534538269,
+      "logits/rejected": -1.0480598211288452,
+      "logps/chosen": -440.39984130859375,
+      "logps/rejected": -503.9505310058594,
+      "loss": 0.6554,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.18013329803943634,
+      "rewards/margins": 0.08887016773223877,
+      "rewards/rejected": -0.2690034508705139,
       "step": 170
     },
     {
       "epoch": 0.37676609105180536,
-      "grad_norm": 2.4425880971749514,
+      "grad_norm": 2.2698140691586306,
       "learning_rate": 3.920161866827889e-07,
-      "logits/chosen": -1.0814361572265625,
-      "logits/rejected": -1.060704231262207,
-      "logps/chosen": -425.4701232910156,
-      "logps/rejected": -474.9969787597656,
-      "loss": 0.6474,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.15264029800891876,
-      "rewards/margins": 0.07376497238874435,
-      "rewards/rejected": -0.22640526294708252,
+      "logits/chosen": -1.1098086833953857,
+      "logits/rejected": -1.0865398645401,
+      "logps/chosen": -460.97833251953125,
+      "logps/rejected": -526.1106567382812,
+      "loss": 0.6414,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.18967969715595245,
+      "rewards/margins": 0.08924828469753265,
+      "rewards/rejected": -0.2789279818534851,
       "step": 180
     },
     {
       "epoch": 0.3976975405546834,
-      "grad_norm": 3.0318995366276247,
+      "grad_norm": 3.600851457498562,
       "learning_rate": 3.765821230985757e-07,
-      "logits/chosen": -1.0653070211410522,
-      "logits/rejected": -1.0440646409988403,
-      "logps/chosen": -428.57293701171875,
-      "logps/rejected": -503.21124267578125,
-      "loss": 0.6487,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.1615397036075592,
-      "rewards/margins": 0.07858813554048538,
-      "rewards/rejected": -0.24012784659862518,
+      "logits/chosen": -1.0811136960983276,
+      "logits/rejected": -1.0507347583770752,
+      "logps/chosen": -491.7823791503906,
+      "logps/rejected": -581.665771484375,
+      "loss": 0.6463,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.22724099457263947,
+      "rewards/margins": 0.09368610382080078,
+      "rewards/rejected": -0.32092708349227905,
       "step": 190
     },
     {
       "epoch": 0.4186289900575615,
-      "grad_norm": 4.194244230703589,
+      "grad_norm": 3.2360530090506083,
       "learning_rate": 3.604695382782159e-07,
-      "logits/chosen": -1.0129735469818115,
-      "logits/rejected": -1.028363585472107,
-      "logps/chosen": -441.21502685546875,
-      "logps/rejected": -554.3182373046875,
-      "loss": 0.6476,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.18880845606327057,
-      "rewards/margins": 0.09274501353502274,
-      "rewards/rejected": -0.2815534770488739,
+      "logits/chosen": -0.986294150352478,
+      "logits/rejected": -1.0004829168319702,
+      "logps/chosen": -555.3709716796875,
+      "logps/rejected": -693.2188720703125,
+      "loss": 0.6419,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.3050481081008911,
+      "rewards/margins": 0.11720240116119385,
+      "rewards/rejected": -0.42225056886672974,
       "step": 200
     },
     {
       "epoch": 0.43956043956043955,
-      "grad_norm": 2.799782012874293,
+      "grad_norm": 3.8624898658875404,
       "learning_rate": 3.4376480090239047e-07,
-      "logits/chosen": -1.0415253639221191,
-      "logits/rejected": -1.0015658140182495,
-      "logps/chosen": -512.2741088867188,
-      "logps/rejected": -587.1564331054688,
-      "loss": 0.6457,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.19915422797203064,
-      "rewards/margins": 0.12847939133644104,
-      "rewards/rejected": -0.3276336193084717,
+      "logits/chosen": -0.9977714419364929,
+      "logits/rejected": -0.9668005108833313,
+      "logps/chosen": -665.3328247070312,
+      "logps/rejected": -751.362060546875,
+      "loss": 0.6487,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3542812466621399,
+      "rewards/margins": 0.13943365216255188,
+      "rewards/rejected": -0.4937148988246918,
       "step": 210
     },
     {
       "epoch": 0.4604918890633176,
-      "grad_norm": 3.38423659902295,
+      "grad_norm": 2.527165628308305,
       "learning_rate": 3.265574537815398e-07,
-      "logits/chosen": -0.9850971102714539,
-      "logits/rejected": -1.0010693073272705,
-      "logps/chosen": -478.1683654785156,
-      "logps/rejected": -623.3740234375,
-      "loss": 0.6426,
+      "logits/chosen": -0.9821435809135437,
+      "logits/rejected": -1.0059399604797363,
+      "logps/chosen": -432.90325927734375,
+      "logps/rejected": -561.0076293945312,
+      "loss": 0.6455,
       "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.24368831515312195,
-      "rewards/margins": 0.13728444278240204,
-      "rewards/rejected": -0.3809727728366852,
+      "rewards/chosen": -0.20027010142803192,
+      "rewards/margins": 0.1195964589715004,
+      "rewards/rejected": -0.3198665678501129,
       "step": 220
     },
     {
       "epoch": 0.48142333856619574,
-      "grad_norm": 2.9823989997956555,
+      "grad_norm": 3.0201938437726894,
       "learning_rate": 3.0893973387735683e-07,
-      "logits/chosen": -1.0038853883743286,
-      "logits/rejected": -0.9878044128417969,
-      "logps/chosen": -531.2225341796875,
-      "logps/rejected": -679.9888916015625,
-      "loss": 0.6402,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.2716134190559387,
-      "rewards/margins": 0.1706905961036682,
-      "rewards/rejected": -0.44230398535728455,
+      "logits/chosen": -0.9755321741104126,
+      "logits/rejected": -0.9638468027114868,
+      "logps/chosen": -537.1400756835938,
+      "logps/rejected": -711.3963012695312,
+      "loss": 0.6351,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.28014427423477173,
+      "rewards/margins": 0.19521525502204895,
+      "rewards/rejected": -0.4753595292568207,
       "step": 230
     },
     {
       "epoch": 0.5023547880690737,
-      "grad_norm": 3.2499709469478084,
+      "grad_norm": 3.5945229999665003,
       "learning_rate": 2.910060778827554e-07,
-      "logits/chosen": -0.9987156987190247,
-      "logits/rejected": -0.9530145525932312,
-      "logps/chosen": -573.1682739257812,
-      "logps/rejected": -688.3665161132812,
-      "loss": 0.6257,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.2874709367752075,
-      "rewards/margins": 0.1330692023038864,
-      "rewards/rejected": -0.4205401539802551,
+      "logits/chosen": -0.9908636212348938,
+      "logits/rejected": -0.9440512657165527,
+      "logps/chosen": -623.9000854492188,
+      "logps/rejected": -765.6077880859375,
+      "loss": 0.6229,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3405396342277527,
+      "rewards/margins": 0.15900567173957825,
+      "rewards/rejected": -0.49954527616500854,
       "step": 240
     },
     {
       "epoch": 0.5232862375719518,
-      "grad_norm": 3.599189967926549,
+      "grad_norm": 4.649933778191512,
       "learning_rate": 2.7285261601056697e-07,
-      "logits/chosen": -0.9661976099014282,
-      "logits/rejected": -0.922343909740448,
-      "logps/chosen": -613.0869140625,
-      "logps/rejected": -763.728515625,
-      "loss": 0.6364,
+      "logits/chosen": -0.981595516204834,
+      "logits/rejected": -0.942014217376709,
+      "logps/chosen": -623.5603637695312,
+      "logps/rejected": -795.4835815429688,
+      "loss": 0.6341,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.33115440607070923,
-      "rewards/margins": 0.19019444286823273,
-      "rewards/rejected": -0.5213488340377808,
+      "rewards/chosen": -0.34366101026535034,
+      "rewards/margins": 0.21082744002342224,
+      "rewards/rejected": -0.5544884204864502,
       "step": 250
     },
     {
       "epoch": 0.54421768707483,
-      "grad_norm": 3.8677130438734486,
+      "grad_norm": 5.029295020454676,
       "learning_rate": 2.5457665670441937e-07,
-      "logits/chosen": -0.9890602231025696,
-      "logits/rejected": -0.987634003162384,
-      "logps/chosen": -588.9368896484375,
-      "logps/rejected": -765.2493286132812,
-      "loss": 0.6281,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.32784414291381836,
-      "rewards/margins": 0.19221974909305573,
-      "rewards/rejected": -0.5200638175010681,
+      "logits/chosen": -1.0086824893951416,
+      "logits/rejected": -1.00519597530365,
+      "logps/chosen": -633.7659912109375,
+      "logps/rejected": -827.9234619140625,
+      "loss": 0.6268,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3744930624961853,
+      "rewards/margins": 0.21027839183807373,
+      "rewards/rejected": -0.584771454334259,
       "step": 260
     },
     {
       "epoch": 0.565149136577708,
-      "grad_norm": 3.5281344720274674,
+      "grad_norm": 4.940462479346541,
       "learning_rate": 2.3627616503391812e-07,
-      "logits/chosen": -0.9270151853561401,
-      "logits/rejected": -0.9179307222366333,
-      "logps/chosen": -658.6036376953125,
-      "logps/rejected": -826.82666015625,
-      "loss": 0.6325,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.35761111974716187,
-      "rewards/margins": 0.19844211637973785,
-      "rewards/rejected": -0.5560532808303833,
+      "logits/chosen": -0.9375359416007996,
+      "logits/rejected": -0.9241452217102051,
+      "logps/chosen": -676.0524291992188,
+      "logps/rejected": -876.9054565429688,
+      "loss": 0.628,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3771703243255615,
+      "rewards/margins": 0.2305041253566742,
+      "rewards/rejected": -0.6076744794845581,
       "step": 270
     },
     {
       "epoch": 0.5860805860805861,
-      "grad_norm": 3.407580559063376,
+      "grad_norm": 3.2120566855593844,
       "learning_rate": 2.1804923757009882e-07,
-      "logits/chosen": -0.8880785703659058,
-      "logits/rejected": -0.8848760724067688,
-      "logps/chosen": -611.5701293945312,
-      "logps/rejected": -751.0750732421875,
-      "loss": 0.6346,
+      "logits/chosen": -0.8946016430854797,
+      "logits/rejected": -0.8984814882278442,
+      "logps/chosen": -645.8856201171875,
+      "logps/rejected": -814.6409301757812,
+      "loss": 0.6274,
       "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.3527330756187439,
-      "rewards/margins": 0.16469557583332062,
-      "rewards/rejected": -0.5174286961555481,
+      "rewards/chosen": -0.3892592191696167,
+      "rewards/margins": 0.19356074929237366,
+      "rewards/rejected": -0.5828199982643127,
       "step": 280
     },
     {
       "epoch": 0.6070120355834642,
-      "grad_norm": 4.11114092580769,
+      "grad_norm": 3.208297566535166,
       "learning_rate": 1.9999357655598891e-07,
-      "logits/chosen": -0.9366544485092163,
-      "logits/rejected": -0.8976960182189941,
-      "logps/chosen": -661.718505859375,
-      "logps/rejected": -826.75146484375,
-      "loss": 0.6346,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.4124836027622223,
-      "rewards/margins": 0.16148509085178375,
-      "rewards/rejected": -0.5739686489105225,
+      "logits/chosen": -0.9358976483345032,
+      "logits/rejected": -0.8927936553955078,
+      "logps/chosen": -712.9257202148438,
+      "logps/rejected": -900.0843505859375,
+      "loss": 0.6289,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.4654994010925293,
+      "rewards/margins": 0.1835438311100006,
+      "rewards/rejected": -0.6490432620048523,
       "step": 290
     },
     {
       "epoch": 0.6279434850863422,
-      "grad_norm": 4.874958720462189,
+      "grad_norm": 4.268524839111709,
       "learning_rate": 1.8220596619089573e-07,
-      "logits/chosen": -0.9249475598335266,
-      "logits/rejected": -0.8965902328491211,
-      "logps/chosen": -754.08154296875,
-      "logps/rejected": -854.07421875,
-      "loss": 0.6167,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.4245515763759613,
-      "rewards/margins": 0.14848145842552185,
-      "rewards/rejected": -0.5730329751968384,
+      "logits/chosen": -0.9108999371528625,
+      "logits/rejected": -0.8883649110794067,
+      "logps/chosen": -782.2093505859375,
+      "logps/rejected": -916.7747802734375,
+      "loss": 0.6109,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.4553016126155853,
+      "rewards/margins": 0.18240997195243835,
+      "rewards/rejected": -0.6377116441726685,
       "step": 300
     },
     {
       "epoch": 0.6488749345892203,
-      "grad_norm": 5.165875801652537,
+      "grad_norm": 4.440153357261806,
       "learning_rate": 1.647817538357072e-07,
-      "logits/chosen": -0.9006611704826355,
-      "logits/rejected": -0.8587985038757324,
-      "logps/chosen": -693.7996826171875,
-      "logps/rejected": -908.5036010742188,
-      "loss": 0.6186,
+      "logits/chosen": -0.8766281008720398,
+      "logits/rejected": -0.8332917094230652,
+      "logps/chosen": -758.6160888671875,
+      "logps/rejected": -1003.10498046875,
+      "loss": 0.6126,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.3858186602592468,
-      "rewards/margins": 0.2699519693851471,
-      "rewards/rejected": -0.6557706594467163,
+      "rewards/chosen": -0.4533390998840332,
+      "rewards/margins": 0.2982035279273987,
+      "rewards/rejected": -0.7515425682067871,
       "step": 310
     },
     {
       "epoch": 0.6698063840920984,
-      "grad_norm": 3.598858462845785,
+      "grad_norm": 4.836923344104393,
       "learning_rate": 1.478143389201113e-07,
-      "logits/chosen": -0.9343263506889343,
-      "logits/rejected": -0.8688840866088867,
-      "logps/chosen": -632.761474609375,
-      "logps/rejected": -885.478515625,
-      "loss": 0.6128,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.3739064931869507,
-      "rewards/margins": 0.2838885486125946,
-      "rewards/rejected": -0.6577950716018677,
+      "logits/chosen": -0.9347564578056335,
+      "logits/rejected": -0.8702303767204285,
+      "logps/chosen": -684.4344482421875,
+      "logps/rejected": -951.7136840820312,
+      "loss": 0.6077,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.42780131101608276,
+      "rewards/margins": 0.29773804545402527,
+      "rewards/rejected": -0.7255394458770752,
       "step": 320
     },
     {
       "epoch": 0.6907378335949764,
-      "grad_norm": 5.102953052593086,
+      "grad_norm": 5.557060773999832,
       "learning_rate": 1.3139467229135998e-07,
-      "logits/chosen": -0.908793568611145,
-      "logits/rejected": -0.8711221814155579,
-      "logps/chosen": -728.1497802734375,
-      "logps/rejected": -968.7476806640625,
-      "loss": 0.6215,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.4502260088920593,
-      "rewards/margins": 0.24815288186073303,
-      "rewards/rejected": -0.6983789205551147,
+      "logits/chosen": -0.8965336680412292,
+      "logits/rejected": -0.8489995002746582,
+      "logps/chosen": -773.0374145507812,
+      "logps/rejected": -1043.8154296875,
+      "loss": 0.616,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4970160126686096,
+      "rewards/margins": 0.27822786569595337,
+      "rewards/rejected": -0.7752438187599182,
       "step": 330
     },
     {
       "epoch": 0.7116692830978545,
-      "grad_norm": 5.671673157970287,
+      "grad_norm": 8.028071802637974,
       "learning_rate": 1.1561076868822755e-07,
-      "logits/chosen": -0.8156379461288452,
-      "logits/rejected": -0.8025031089782715,
-      "logps/chosen": -742.4412841796875,
-      "logps/rejected": -978.5466918945312,
-      "loss": 0.6096,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.44500136375427246,
-      "rewards/margins": 0.2643851637840271,
-      "rewards/rejected": -0.7093865275382996,
+      "logits/chosen": -0.7840911746025085,
+      "logits/rejected": -0.7743405103683472,
+      "logps/chosen": -797.73974609375,
+      "logps/rejected": -1051.7327880859375,
+      "loss": 0.6039,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5024133324623108,
+      "rewards/margins": 0.2816846966743469,
+      "rewards/rejected": -0.7840980291366577,
       "step": 340
     },
     {
       "epoch": 0.7326007326007326,
-      "grad_norm": 4.788572684554985,
+      "grad_norm": 5.482245632746445,
       "learning_rate": 1.0054723495346482e-07,
-      "logits/chosen": -0.8821672201156616,
-      "logits/rejected": -0.8667781949043274,
-      "logps/chosen": -744.6392822265625,
-      "logps/rejected": -950.1046142578125,
-      "loss": 0.6065,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.4801097810268402,
-      "rewards/margins": 0.23577599227428436,
-      "rewards/rejected": -0.7158857583999634,
+      "logits/chosen": -0.8621054887771606,
+      "logits/rejected": -0.8484535217285156,
+      "logps/chosen": -779.8978881835938,
+      "logps/rejected": -1020.26513671875,
+      "loss": 0.5996,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5172659158706665,
+      "rewards/margins": 0.27023959159851074,
+      "rewards/rejected": -0.7875055074691772,
       "step": 350
     },
     {
       "epoch": 0.7535321821036107,
-      "grad_norm": 4.756088898998946,
+      "grad_norm": 4.61428054230484,
       "learning_rate": 8.628481651367875e-08,
-      "logits/chosen": -0.8794587850570679,
-      "logits/rejected": -0.8525463342666626,
-      "logps/chosen": -842.9227294921875,
-      "logps/rejected": -1010.73828125,
-      "loss": 0.6269,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.528265655040741,
-      "rewards/margins": 0.20947960019111633,
-      "rewards/rejected": -0.7377452254295349,
+      "logits/chosen": -0.8745294809341431,
+      "logits/rejected": -0.8429532051086426,
+      "logps/chosen": -868.0397338867188,
+      "logps/rejected": -1069.2259521484375,
+      "loss": 0.6226,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5554712414741516,
+      "rewards/margins": 0.24238689243793488,
+      "rewards/rejected": -0.7978580594062805,
       "step": 360
     },
     {
       "epoch": 0.7744636316064888,
-      "grad_norm": 4.287691995792507,
+      "grad_norm": 4.012064999162859,
       "learning_rate": 7.289996455765748e-08,
-      "logits/chosen": -0.9290102124214172,
-      "logits/rejected": -0.8693262338638306,
-      "logps/chosen": -681.6297607421875,
-      "logps/rejected": -900.4710083007812,
-      "loss": 0.6112,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.41641873121261597,
-      "rewards/margins": 0.2501312494277954,
-      "rewards/rejected": -0.6665499806404114,
+      "logits/chosen": -0.9377773404121399,
+      "logits/rejected": -0.8822382688522339,
+      "logps/chosen": -725.4534912109375,
+      "logps/rejected": -960.8234252929688,
+      "loss": 0.6084,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.462857186794281,
+      "rewards/margins": 0.2657161355018616,
+      "rewards/rejected": -0.7285734415054321,
       "step": 370
     },
     {
       "epoch": 0.7953950811093669,
-      "grad_norm": 3.351788727441381,
+      "grad_norm": 3.5554616562598995,
       "learning_rate": 6.046442623320145e-08,
-      "logits/chosen": -0.8099156618118286,
-      "logits/rejected": -0.7845919728279114,
-      "logps/chosen": -751.2888793945312,
-      "logps/rejected": -1116.771484375,
-      "loss": 0.6152,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.5037558674812317,
-      "rewards/margins": 0.3494088351726532,
-      "rewards/rejected": -0.8531646728515625,
+      "logits/chosen": -0.7937309145927429,
+      "logits/rejected": -0.766541063785553,
+      "logps/chosen": -826.75537109375,
+      "logps/rejected": -1213.797607421875,
+      "loss": 0.6092,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5811253786087036,
+      "rewards/margins": 0.37074214220046997,
+      "rewards/rejected": -0.951867401599884,
       "step": 380
     },
     {
       "epoch": 0.8163265306122449,
-      "grad_norm": 4.2822185775858355,
+      "grad_norm": 5.133888886461569,
       "learning_rate": 4.904486005914027e-08,
-      "logits/chosen": -0.8926030993461609,
-      "logits/rejected": -0.8195274472236633,
-      "logps/chosen": -821.78125,
-      "logps/rejected": -1080.7801513671875,
-      "loss": 0.6109,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.4835873544216156,
-      "rewards/margins": 0.2913810908794403,
-      "rewards/rejected": -0.7749683856964111,
+      "logits/chosen": -0.8645860552787781,
+      "logits/rejected": -0.7781597971916199,
+      "logps/chosen": -887.5338745117188,
+      "logps/rejected": -1190.846923828125,
+      "loss": 0.6032,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5518342852592468,
+      "rewards/margins": 0.3350018858909607,
+      "rewards/rejected": -0.8868362307548523,
       "step": 390
     },
     {
       "epoch": 0.837257980115123,
-      "grad_norm": 3.832733560631146,
+      "grad_norm": 5.1958119384493875,
       "learning_rate": 3.8702478614051345e-08,
-      "logits/chosen": -0.8491037487983704,
-      "logits/rejected": -0.8279684782028198,
-      "logps/chosen": -704.4039916992188,
-      "logps/rejected": -935.1871337890625,
-      "loss": 0.6164,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.44019538164138794,
-      "rewards/margins": 0.25378626585006714,
-      "rewards/rejected": -0.6939816474914551,
+      "logits/chosen": -0.8455416560173035,
+      "logits/rejected": -0.8282378911972046,
+      "logps/chosen": -726.9100952148438,
+      "logps/rejected": -962.5436401367188,
+      "loss": 0.6137,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.46447521448135376,
+      "rewards/margins": 0.25824955105781555,
+      "rewards/rejected": -0.7227246761322021,
       "step": 400
     },
     {
       "epoch": 0.858189429618001,
-      "grad_norm": 3.767221913490684,
+      "grad_norm": 5.087694596010447,
       "learning_rate": 2.9492720416985e-08,
-      "logits/chosen": -0.8924685716629028,
-      "logits/rejected": -0.858057975769043,
-      "logps/chosen": -775.3809814453125,
-      "logps/rejected": -1018.1121215820312,
-      "loss": 0.6163,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.46508699655532837,
-      "rewards/margins": 0.28694623708724976,
-      "rewards/rejected": -0.7520332932472229,
+      "logits/chosen": -0.881341278553009,
+      "logits/rejected": -0.8477126359939575,
+      "logps/chosen": -797.9344482421875,
+      "logps/rejected": -1068.551513671875,
+      "loss": 0.6109,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4897310137748718,
+      "rewards/margins": 0.31405144929885864,
+      "rewards/rejected": -0.8037824630737305,
       "step": 410
     },
     {
       "epoch": 0.8791208791208791,
-      "grad_norm": 5.174199687247601,
+      "grad_norm": 4.7011066546022855,
       "learning_rate": 2.1464952759020856e-08,
-      "logits/chosen": -0.7606777548789978,
-      "logits/rejected": -0.7737725377082825,
-      "logps/chosen": -774.3548583984375,
-      "logps/rejected": -1066.3587646484375,
-      "loss": 0.6037,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.5085434317588806,
-      "rewards/margins": 0.28706783056259155,
-      "rewards/rejected": -0.7956112623214722,
+      "logits/chosen": -0.7331596612930298,
+      "logits/rejected": -0.7492183446884155,
+      "logps/chosen": -795.1165161132812,
+      "logps/rejected": -1116.110595703125,
+      "loss": 0.5976,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.5313975811004639,
+      "rewards/margins": 0.31534695625305176,
+      "rewards/rejected": -0.8467445373535156,
       "step": 420
     },
     {
       "epoch": 0.9000523286237572,
-      "grad_norm": 3.6763407260644145,
+      "grad_norm": 4.174114432694521,
       "learning_rate": 1.4662207078575684e-08,
-      "logits/chosen": -0.8245135545730591,
-      "logits/rejected": -0.7580205202102661,
-      "logps/chosen": -801.9810791015625,
-      "logps/rejected": -1054.1705322265625,
-      "loss": 0.6119,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.5072224736213684,
-      "rewards/margins": 0.2761218249797821,
-      "rewards/rejected": -0.7833442091941833,
+      "logits/chosen": -0.8007818460464478,
+      "logits/rejected": -0.7300530672073364,
+      "logps/chosen": -845.8348388671875,
+      "logps/rejected": -1122.674072265625,
+      "loss": 0.6044,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5535520315170288,
+      "rewards/margins": 0.30010417103767395,
+      "rewards/rejected": -0.8536561131477356,
       "step": 430
     },
     {
       "epoch": 0.9209837781266352,
-      "grad_norm": 3.9565073368902963,
+      "grad_norm": 4.573763510440249,
       "learning_rate": 9.12094829893642e-09,
-      "logits/chosen": -0.8855506777763367,
-      "logits/rejected": -0.8454933166503906,
-      "logps/chosen": -754.6903076171875,
-      "logps/rejected": -885.1017456054688,
-      "loss": 0.6043,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.4870569705963135,
-      "rewards/margins": 0.17229454219341278,
-      "rewards/rejected": -0.6593514680862427,
+      "logits/chosen": -0.8669272661209106,
+      "logits/rejected": -0.8168144226074219,
+      "logps/chosen": -778.4366455078125,
+      "logps/rejected": -945.3597412109375,
+      "loss": 0.596,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.5131410956382751,
+      "rewards/margins": 0.20753808319568634,
+      "rewards/rejected": -0.7206791639328003,
       "step": 440
     },
     {
       "epoch": 0.9419152276295133,
-      "grad_norm": 3.701479740364913,
+      "grad_norm": 5.324804725259421,
       "learning_rate": 4.8708793644441086e-09,
-      "logits/chosen": -0.7477255463600159,
-      "logits/rejected": -0.7270427346229553,
-      "logps/chosen": -780.8983154296875,
-      "logps/rejected": -1019.4451293945312,
-      "loss": 0.6124,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.5077296495437622,
-      "rewards/margins": 0.251300185918808,
-      "rewards/rejected": -0.7590298056602478,
+      "logits/chosen": -0.7154260873794556,
+      "logits/rejected": -0.6910079121589661,
+      "logps/chosen": -808.1412963867188,
+      "logps/rejected": -1094.1324462890625,
+      "loss": 0.6039,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.5372119545936584,
+      "rewards/margins": 0.29753977060317993,
+      "rewards/rejected": -0.8347517251968384,
       "step": 450
     },
     {
       "epoch": 0.9628466771323915,
-      "grad_norm": 5.067825971160372,
+      "grad_norm": 4.938304952484282,
       "learning_rate": 1.9347820230782295e-09,
-      "logits/chosen": -0.7995008230209351,
-      "logits/rejected": -0.7618998289108276,
-      "logps/chosen": -767.2562866210938,
-      "logps/rejected": -937.3308715820312,
-      "loss": 0.6147,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.49197807908058167,
-      "rewards/margins": 0.2139851152896881,
-      "rewards/rejected": -0.705963134765625,
+      "logits/chosen": -0.7675133943557739,
+      "logits/rejected": -0.7233086228370667,
+      "logps/chosen": -805.8953857421875,
+      "logps/rejected": -1008.8405151367188,
+      "loss": 0.6124,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.5324074625968933,
+      "rewards/margins": 0.2462809532880783,
+      "rewards/rejected": -0.7786884307861328,
       "step": 460
     },
     {
       "epoch": 0.9837781266352695,
-      "grad_norm": 5.987105760607766,
+      "grad_norm": 7.346736713965509,
       "learning_rate": 3.2839470889836627e-10,
-      "logits/chosen": -0.8542754054069519,
-      "logits/rejected": -0.8349153399467468,
-      "logps/chosen": -772.111328125,
-      "logps/rejected": -954.55517578125,
-      "loss": 0.6047,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.47330260276794434,
-      "rewards/margins": 0.19964265823364258,
-      "rewards/rejected": -0.6729452610015869,
+      "logits/chosen": -0.8261724710464478,
+      "logits/rejected": -0.8037177324295044,
+      "logps/chosen": -807.8523559570312,
+      "logps/rejected": -1025.576904296875,
+      "loss": 0.5979,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5114091634750366,
+      "rewards/margins": 0.23410066962242126,
+      "rewards/rejected": -0.7455097436904907,
       "step": 470
     },
     {
       "epoch": 0.9984301412872841,
-      "eval_logits/chosen": -0.8622139692306519,
-      "eval_logits/rejected": -0.8340398669242859,
-      "eval_logps/chosen": -740.609375,
-      "eval_logps/rejected": -1076.8990478515625,
-      "eval_loss": 0.6172269582748413,
-      "eval_rewards/accuracies": 0.75,
-      "eval_rewards/chosen": -0.46546465158462524,
-      "eval_rewards/margins": 0.33346831798553467,
-      "eval_rewards/rejected": -0.7989329099655151,
-      "eval_runtime": 191.0283,
-      "eval_samples_per_second": 10.47,
-      "eval_steps_per_second": 0.168,
+      "eval_logits/chosen": -0.8428901433944702,
+      "eval_logits/rejected": -0.8125907778739929,
+      "eval_logps/chosen": -782.0501098632812,
+      "eval_logps/rejected": -1150.804931640625,
+      "eval_loss": 0.6099801063537598,
+      "eval_rewards/accuracies": 0.73828125,
+      "eval_rewards/chosen": -0.5090480446815491,
+      "eval_rewards/margins": 0.3657650351524353,
+      "eval_rewards/rejected": -0.8748130202293396,
+      "eval_runtime": 245.867,
+      "eval_samples_per_second": 8.134,
+      "eval_steps_per_second": 0.13,
       "step": 477
     },
     {
       "epoch": 0.9984301412872841,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.6437127205310877,
-      "train_runtime": 13588.3404,
-      "train_samples_per_second": 4.499,
+      "train_loss": 0.6406088435175034,
+      "train_runtime": 13597.1074,
+      "train_samples_per_second": 4.496,
       "train_steps_per_second": 0.035
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c528bf9faa59f5082bdd7cb4aa549d795a0bd83e24252c64269557f86a809bf
+oid sha256:9591842ca276398129eb2cc4ab6105a079ba15685fc14f478d5d0f816a598cdb
 size 7672