Model save

README.md

@@ -16,15 +16,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4929
-- Rewards/chosen: -1.1416
-- Rewards/rejected: -2.1919
-- Rewards/accuracies: 0.7852
-- Rewards/margins: 1.0502
-- Logps/rejected: -469.4702
-- Logps/chosen: -363.3538
-- Logits/rejected: -1.6742
-- Logits/chosen: -1.7399
+- Loss: 0.5132
+- Rewards/chosen: -0.8911
+- Rewards/rejected: -1.7562
+- Rewards/accuracies: 0.7812
+- Rewards/margins: 0.8651
+- Logps/rejected: -432.3867
+- Logps/chosen: -346.3244
+- Logits/rejected: -2.4131
+- Logits/chosen: -2.4336
 
 ## Model description
 
@@ -61,12 +61,12 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4838        | 0.9984 | 477  | 0.4929          | -1.1416        | -2.1919          | 0.7852             | 1.0502          | -469.4702      | -363.3538    | -1.6742         | -1.7399       |
+| 0.4975        | 0.9984 | 477  | 0.5132          | -0.8911        | -1.7562          | 0.7812             | 0.8651          | -432.3867      | -346.3244    | -2.4131         | -2.4336       |
 
 
 ### Framework versions
 
 - Transformers 4.45.2
 - Pytorch 2.5.1+rocm6.2
-- Datasets 3.5.0
+- Datasets 3.2.0
 - Tokenizers 0.20.3

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.5343590847351266,
-    "train_runtime": 13524.9429,
+    "train_loss": 0.5516081291174738,
+    "train_runtime": 26908.0955,
     "train_samples": 61134,
-    "train_samples_per_second": 4.52,
-    "train_steps_per_second": 0.035
+    "train_samples_per_second": 2.272,
+    "train_steps_per_second": 0.018
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.5343590847351266,
-    "train_runtime": 13524.9429,
+    "train_loss": 0.5516081291174738,
+    "train_runtime": 26908.0955,
     "train_samples": 61134,
-    "train_samples_per_second": 4.52,
-    "train_steps_per_second": 0.035
+    "train_samples_per_second": 2.272,
+    "train_steps_per_second": 0.018
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0020931449502878076,
-      "grad_norm": 9.580024156968456,
+      "grad_norm": 8.706076728432208,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -2.430640935897827,
-      "logits/rejected": -2.325669050216675,
-      "logps/chosen": -301.1888122558594,
-      "logps/rejected": -346.8065185546875,
+      "logits/chosen": -2.9199132919311523,
+      "logits/rejected": -2.8931403160095215,
+      "logps/chosen": -309.4719543457031,
+      "logps/rejected": -355.0931396484375,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,733 +25,733 @@
     },
     {
       "epoch": 0.020931449502878074,
-      "grad_norm": 9.535503163597756,
+      "grad_norm": 8.811125958442949,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.121445655822754,
-      "logits/rejected": -2.055097818374634,
-      "logps/chosen": -287.1705322265625,
-      "logps/rejected": -259.10406494140625,
+      "logits/chosen": -2.7454538345336914,
+      "logits/rejected": -2.7216293811798096,
+      "logps/chosen": -295.4472351074219,
+      "logps/rejected": -266.0006103515625,
       "loss": 0.6932,
-      "rewards/accuracies": 0.4652777910232544,
-      "rewards/chosen": 0.00018805032595992088,
-      "rewards/margins": 0.00044798923772759736,
-      "rewards/rejected": -0.00025993891176767647,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": 0.0005748738767579198,
+      "rewards/margins": 0.0004018136824015528,
+      "rewards/rejected": 0.0001730601507006213,
       "step": 10
     },
     {
       "epoch": 0.04186289900575615,
-      "grad_norm": 9.135166094203631,
+      "grad_norm": 8.356417875630138,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.2380423545837402,
-      "logits/rejected": -2.1180529594421387,
-      "logps/chosen": -291.8775939941406,
-      "logps/rejected": -266.7220764160156,
-      "loss": 0.6918,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 0.0038302987813949585,
-      "rewards/margins": 0.002486014971509576,
-      "rewards/rejected": 0.0013442838098853827,
+      "logits/chosen": -2.8119170665740967,
+      "logits/rejected": -2.7720179557800293,
+      "logps/chosen": -300.11346435546875,
+      "logps/rejected": -273.6678161621094,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.002672016154974699,
+      "rewards/margins": 0.0024929677601903677,
+      "rewards/rejected": 0.00017904839478433132,
       "step": 20
     },
     {
       "epoch": 0.06279434850863422,
-      "grad_norm": 9.156841604795108,
+      "grad_norm": 8.3364028133132,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.204575300216675,
-      "logits/rejected": -2.0733189582824707,
-      "logps/chosen": -273.11627197265625,
-      "logps/rejected": -233.1331024169922,
-      "loss": 0.6858,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.02578217163681984,
-      "rewards/margins": 0.014600785449147224,
-      "rewards/rejected": 0.011181383393704891,
+      "logits/chosen": -2.7947373390197754,
+      "logits/rejected": -2.7448770999908447,
+      "logps/chosen": -283.2146301269531,
+      "logps/rejected": -239.5857391357422,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.014546801336109638,
+      "rewards/margins": 0.008760047145187855,
+      "rewards/rejected": 0.005786753259599209,
       "step": 30
     },
     {
       "epoch": 0.0837257980115123,
-      "grad_norm": 9.688914936946615,
+      "grad_norm": 8.740777113133271,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.090651750564575,
-      "logits/rejected": -2.000094413757324,
-      "logps/chosen": -247.9547119140625,
-      "logps/rejected": -251.52322387695312,
-      "loss": 0.6676,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.017582237720489502,
-      "rewards/margins": 0.06143052503466606,
-      "rewards/rejected": -0.04384828731417656,
+      "logits/chosen": -2.7081985473632812,
+      "logits/rejected": -2.6920669078826904,
+      "logps/chosen": -253.94091796875,
+      "logps/rejected": -253.13613891601562,
+      "loss": 0.6779,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.028159379959106445,
+      "rewards/margins": 0.03365633636713028,
+      "rewards/rejected": -0.005496961064636707,
       "step": 40
     },
     {
       "epoch": 0.10465724751439037,
-      "grad_norm": 11.544993249044957,
+      "grad_norm": 8.967527533592596,
       "learning_rate": 4.999731868769026e-07,
-      "logits/chosen": -2.148649215698242,
-      "logits/rejected": -2.025043249130249,
-      "logps/chosen": -275.79974365234375,
-      "logps/rejected": -283.02716064453125,
-      "loss": 0.6423,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.13076314330101013,
-      "rewards/margins": 0.11921179294586182,
-      "rewards/rejected": -0.24997496604919434,
+      "logits/chosen": -2.727299451828003,
+      "logits/rejected": -2.6693475246429443,
+      "logps/chosen": -271.21044921875,
+      "logps/rejected": -271.84075927734375,
+      "loss": 0.6595,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.005260963924229145,
+      "rewards/margins": 0.06768764555454254,
+      "rewards/rejected": -0.07294861972332001,
       "step": 50
     },
     {
       "epoch": 0.12558869701726844,
-      "grad_norm": 15.350321140435819,
+      "grad_norm": 10.674874943980884,
       "learning_rate": 4.990353313429303e-07,
-      "logits/chosen": -2.2131295204162598,
-      "logits/rejected": -2.1153783798217773,
-      "logps/chosen": -274.0061950683594,
-      "logps/rejected": -293.47113037109375,
-      "loss": 0.616,
+      "logits/chosen": -2.7367279529571533,
+      "logits/rejected": -2.695169448852539,
+      "logps/chosen": -257.17059326171875,
+      "logps/rejected": -261.27813720703125,
+      "loss": 0.6389,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.34647348523139954,
-      "rewards/margins": 0.27237194776535034,
-      "rewards/rejected": -0.6188454031944275,
+      "rewards/chosen": -0.1086881160736084,
+      "rewards/margins": 0.14326919615268707,
+      "rewards/rejected": -0.25195732712745667,
       "step": 60
     },
     {
       "epoch": 0.14652014652014653,
-      "grad_norm": 17.446151967081736,
+      "grad_norm": 13.371926053620712,
       "learning_rate": 4.967625656594781e-07,
-      "logits/chosen": -2.153040647506714,
-      "logits/rejected": -2.098363161087036,
-      "logps/chosen": -332.89813232421875,
-      "logps/rejected": -358.3309326171875,
-      "loss": 0.5867,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.5407691597938538,
-      "rewards/margins": 0.38723263144493103,
-      "rewards/rejected": -0.9280017614364624,
+      "logits/chosen": -2.677611827850342,
+      "logits/rejected": -2.6540932655334473,
+      "logps/chosen": -314.21636962890625,
+      "logps/rejected": -325.9127502441406,
+      "loss": 0.6064,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.27906379103660583,
+      "rewards/margins": 0.2486257255077362,
+      "rewards/rejected": -0.527689516544342,
       "step": 70
     },
     {
       "epoch": 0.1674515960230246,
-      "grad_norm": 27.42185029612204,
+      "grad_norm": 20.543041884620823,
       "learning_rate": 4.93167072587771e-07,
-      "logits/chosen": -2.3443994522094727,
-      "logits/rejected": -2.1753482818603516,
-      "logps/chosen": -361.7871398925781,
-      "logps/rejected": -336.57354736328125,
-      "loss": 0.5807,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.5057138800621033,
-      "rewards/margins": 0.43273383378982544,
-      "rewards/rejected": -0.9384476542472839,
+      "logits/chosen": -2.734398126602173,
+      "logits/rejected": -2.669373035430908,
+      "logps/chosen": -364.68597412109375,
+      "logps/rejected": -331.4052734375,
+      "loss": 0.5927,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.43884754180908203,
+      "rewards/margins": 0.39294886589050293,
+      "rewards/rejected": -0.8317963480949402,
       "step": 80
     },
     {
       "epoch": 0.18838304552590268,
-      "grad_norm": 22.058501406046492,
+      "grad_norm": 23.3331824272025,
       "learning_rate": 4.882681251368548e-07,
-      "logits/chosen": -2.261949062347412,
-      "logits/rejected": -2.188600778579712,
-      "logps/chosen": -303.0185241699219,
-      "logps/rejected": -365.95440673828125,
-      "loss": 0.5512,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.6372281312942505,
-      "rewards/margins": 0.5772984623908997,
-      "rewards/rejected": -1.2145264148712158,
+      "logits/chosen": -2.672283887863159,
+      "logits/rejected": -2.6232357025146484,
+      "logps/chosen": -320.28662109375,
+      "logps/rejected": -377.03521728515625,
+      "loss": 0.5607,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.7396225929260254,
+      "rewards/margins": 0.5407915711402893,
+      "rewards/rejected": -1.2804142236709595,
       "step": 90
     },
     {
       "epoch": 0.20931449502878074,
-      "grad_norm": 23.10731770703464,
+      "grad_norm": 25.668798919400494,
       "learning_rate": 4.820919832540181e-07,
-      "logits/chosen": -2.2352817058563232,
-      "logits/rejected": -2.153383255004883,
-      "logps/chosen": -351.9589538574219,
-      "logps/rejected": -398.7929992675781,
-      "loss": 0.5536,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.7218375205993652,
-      "rewards/margins": 0.6266552805900574,
-      "rewards/rejected": -1.3484928607940674,
+      "logits/chosen": -2.6326744556427,
+      "logits/rejected": -2.6118311882019043,
+      "logps/chosen": -355.27789306640625,
+      "logps/rejected": -396.7851867675781,
+      "loss": 0.5661,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.6812170743942261,
+      "rewards/margins": 0.5840116143226624,
+      "rewards/rejected": -1.2652287483215332,
       "step": 100
     },
     {
       "epoch": 0.2302459445316588,
-      "grad_norm": 26.246079226878386,
+      "grad_norm": 26.120384615402926,
       "learning_rate": 4.7467175306295647e-07,
-      "logits/chosen": -2.2616801261901855,
-      "logits/rejected": -2.175260066986084,
-      "logps/chosen": -344.7423400878906,
-      "logps/rejected": -380.02166748046875,
-      "loss": 0.5621,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.6688632369041443,
-      "rewards/margins": 0.5221593976020813,
-      "rewards/rejected": -1.1910226345062256,
+      "logits/chosen": -2.6640563011169434,
+      "logits/rejected": -2.618682384490967,
+      "logps/chosen": -367.52850341796875,
+      "logps/rejected": -391.9440612792969,
+      "loss": 0.5878,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8145754933357239,
+      "rewards/margins": 0.4289913773536682,
+      "rewards/rejected": -1.243566870689392,
       "step": 110
     },
     {
       "epoch": 0.25117739403453687,
-      "grad_norm": 24.081738347134376,
+      "grad_norm": 21.77135334354446,
       "learning_rate": 4.6604720940421207e-07,
-      "logits/chosen": -2.1112895011901855,
-      "logits/rejected": -2.0991122722625732,
-      "logps/chosen": -323.6635437011719,
-      "logps/rejected": -379.61480712890625,
-      "loss": 0.5492,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.6999741792678833,
-      "rewards/margins": 0.6201815009117126,
-      "rewards/rejected": -1.3201555013656616,
+      "logits/chosen": -2.555119037628174,
+      "logits/rejected": -2.56174898147583,
+      "logps/chosen": -314.50299072265625,
+      "logps/rejected": -358.1918029785156,
+      "loss": 0.5653,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.5376996397972107,
+      "rewards/margins": 0.5146290063858032,
+      "rewards/rejected": -1.0523287057876587,
       "step": 120
     },
     {
       "epoch": 0.272108843537415,
-      "grad_norm": 23.369346780966698,
+      "grad_norm": 33.88547905386826,
       "learning_rate": 4.5626458262912735e-07,
-      "logits/chosen": -2.1512370109558105,
-      "logits/rejected": -2.108196973800659,
-      "logps/chosen": -335.7061767578125,
-      "logps/rejected": -375.25469970703125,
-      "loss": 0.5373,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.7045310139656067,
-      "rewards/margins": 0.5020760297775269,
-      "rewards/rejected": -1.2066071033477783,
+      "logits/chosen": -2.587398052215576,
+      "logits/rejected": -2.5758957862854004,
+      "logps/chosen": -325.7637939453125,
+      "logps/rejected": -355.89044189453125,
+      "loss": 0.5484,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.5277008414268494,
+      "rewards/margins": 0.4236302375793457,
+      "rewards/rejected": -0.9513310194015503,
       "step": 130
     },
     {
       "epoch": 0.29304029304029305,
-      "grad_norm": 20.81429209370454,
+      "grad_norm": 31.071210271040595,
       "learning_rate": 4.453763107901675e-07,
-      "logits/chosen": -2.191638469696045,
-      "logits/rejected": -2.123680591583252,
-      "logps/chosen": -364.34161376953125,
-      "logps/rejected": -393.08013916015625,
-      "loss": 0.5473,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.6576336622238159,
-      "rewards/margins": 0.6938663125038147,
-      "rewards/rejected": -1.3515000343322754,
+      "logits/chosen": -2.615340232849121,
+      "logits/rejected": -2.5916059017181396,
+      "logps/chosen": -365.6969299316406,
+      "logps/rejected": -390.7157287597656,
+      "loss": 0.5565,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.5825755596160889,
+      "rewards/margins": 0.6785892248153687,
+      "rewards/rejected": -1.261164903640747,
       "step": 140
     },
     {
       "epoch": 0.3139717425431711,
-      "grad_norm": 29.1079724061151,
+      "grad_norm": 21.469144106917998,
       "learning_rate": 4.3344075855595097e-07,
-      "logits/chosen": -2.207198143005371,
-      "logits/rejected": -2.1470441818237305,
-      "logps/chosen": -336.94122314453125,
-      "logps/rejected": -356.14520263671875,
-      "loss": 0.5473,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.7152302861213684,
-      "rewards/margins": 0.504962682723999,
-      "rewards/rejected": -1.2201929092407227,
+      "logits/chosen": -2.5812668800354004,
+      "logits/rejected": -2.554624557495117,
+      "logps/chosen": -355.30401611328125,
+      "logps/rejected": -370.72967529296875,
+      "loss": 0.5675,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8197686076164246,
+      "rewards/margins": 0.48544248938560486,
+      "rewards/rejected": -1.305211067199707,
       "step": 150
     },
     {
       "epoch": 0.3349031920460492,
-      "grad_norm": 27.917087192433524,
+      "grad_norm": 21.777071374240855,
       "learning_rate": 4.2052190435769554e-07,
-      "logits/chosen": -2.2518768310546875,
-      "logits/rejected": -2.1720962524414062,
-      "logps/chosen": -333.5887756347656,
-      "logps/rejected": -383.0411682128906,
-      "loss": 0.5556,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.7204815745353699,
-      "rewards/margins": 0.7067698240280151,
-      "rewards/rejected": -1.4272514581680298,
+      "logits/chosen": -2.6186599731445312,
+      "logits/rejected": -2.581908941268921,
+      "logps/chosen": -337.18695068359375,
+      "logps/rejected": -372.72894287109375,
+      "loss": 0.5668,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.6758376359939575,
+      "rewards/margins": 0.6010496020317078,
+      "rewards/rejected": -1.2768871784210205,
       "step": 160
     },
     {
       "epoch": 0.35583464154892724,
-      "grad_norm": 28.43046248619016,
+      "grad_norm": 19.375383016314835,
       "learning_rate": 4.0668899744407567e-07,
-      "logits/chosen": -2.0816593170166016,
-      "logits/rejected": -2.050276041030884,
-      "logps/chosen": -338.7085876464844,
-      "logps/rejected": -370.1266174316406,
-      "loss": 0.5389,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.9299048185348511,
-      "rewards/margins": 0.5225407481193542,
-      "rewards/rejected": -1.45244562625885,
+      "logits/chosen": -2.480734348297119,
+      "logits/rejected": -2.470438241958618,
+      "logps/chosen": -332.8466796875,
+      "logps/rejected": -356.3028259277344,
+      "loss": 0.56,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7997764348983765,
+      "rewards/margins": 0.4615844786167145,
+      "rewards/rejected": -1.261361002922058,
       "step": 170
     },
     {
       "epoch": 0.37676609105180536,
-      "grad_norm": 27.829977826652623,
+      "grad_norm": 20.990946288367706,
       "learning_rate": 3.920161866827889e-07,
-      "logits/chosen": -2.0916779041290283,
-      "logits/rejected": -2.0410423278808594,
-      "logps/chosen": -364.9996643066406,
-      "logps/rejected": -400.76239013671875,
-      "loss": 0.5074,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.081892967224121,
-      "rewards/margins": 0.6016584038734436,
-      "rewards/rejected": -1.6835514307022095,
+      "logits/chosen": -2.5555384159088135,
+      "logits/rejected": -2.530357599258423,
+      "logps/chosen": -356.4952087402344,
+      "logps/rejected": -373.42803955078125,
+      "loss": 0.5268,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.9372851252555847,
+      "rewards/margins": 0.4233158528804779,
+      "rewards/rejected": -1.3606010675430298,
       "step": 180
     },
     {
       "epoch": 0.3976975405546834,
-      "grad_norm": 27.60089876506741,
+      "grad_norm": 22.804559593658382,
       "learning_rate": 3.765821230985757e-07,
-      "logits/chosen": -2.0626139640808105,
-      "logits/rejected": -2.0298991203308105,
-      "logps/chosen": -340.3715515136719,
-      "logps/rejected": -395.8924255371094,
-      "loss": 0.5326,
+      "logits/chosen": -2.4814000129699707,
+      "logits/rejected": -2.4526283740997314,
+      "logps/chosen": -335.11944580078125,
+      "logps/rejected": -378.43768310546875,
+      "loss": 0.55,
       "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.9407373666763306,
-      "rewards/margins": 0.6121485829353333,
-      "rewards/rejected": -1.5528860092163086,
+      "rewards/chosen": -0.8277766108512878,
+      "rewards/margins": 0.4885457158088684,
+      "rewards/rejected": -1.3163223266601562,
       "step": 190
     },
     {
       "epoch": 0.4186289900575615,
-      "grad_norm": 38.5217107381582,
+      "grad_norm": 43.87788468421851,
       "learning_rate": 3.604695382782159e-07,
-      "logits/chosen": -2.0065114498138428,
-      "logits/rejected": -2.0017216205596924,
-      "logps/chosen": -332.4302673339844,
-      "logps/rejected": -414.33251953125,
-      "loss": 0.5384,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.9781379699707031,
-      "rewards/margins": 0.6036709547042847,
-      "rewards/rejected": -1.5818089246749878,
+      "logits/chosen": -2.4679489135742188,
+      "logits/rejected": -2.4767565727233887,
+      "logps/chosen": -312.5929260253906,
+      "logps/rejected": -386.0437316894531,
+      "loss": 0.557,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.7163432836532593,
+      "rewards/margins": 0.5137871503829956,
+      "rewards/rejected": -1.2301304340362549,
       "step": 200
     },
     {
       "epoch": 0.43956043956043955,
-      "grad_norm": 25.593502267183467,
+      "grad_norm": 23.27261695682354,
       "learning_rate": 3.4376480090239047e-07,
-      "logits/chosen": -2.039764881134033,
-      "logits/rejected": -1.9067729711532593,
-      "logps/chosen": -379.778076171875,
-      "logps/rejected": -392.64239501953125,
-      "loss": 0.5337,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.8867334127426147,
-      "rewards/margins": 0.6549147367477417,
-      "rewards/rejected": -1.5416481494903564,
+      "logits/chosen": -2.5316569805145264,
+      "logits/rejected": -2.4835801124572754,
+      "logps/chosen": -373.7596740722656,
+      "logps/rejected": -372.31744384765625,
+      "loss": 0.554,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.734451413154602,
+      "rewards/margins": 0.5430240631103516,
+      "rewards/rejected": -1.2774755954742432,
       "step": 210
     },
     {
       "epoch": 0.4604918890633176,
-      "grad_norm": 36.182958654851035,
+      "grad_norm": 23.03363112192297,
       "learning_rate": 3.265574537815398e-07,
-      "logits/chosen": -1.8658435344696045,
-      "logits/rejected": -1.8497034311294556,
-      "logps/chosen": -316.08392333984375,
-      "logps/rejected": -390.88018798828125,
-      "loss": 0.5137,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.9008970260620117,
-      "rewards/margins": 0.7025474309921265,
-      "rewards/rejected": -1.6034444570541382,
+      "logits/chosen": -2.44421648979187,
+      "logits/rejected": -2.4709441661834717,
+      "logps/chosen": -309.4106750488281,
+      "logps/rejected": -372.0838928222656,
+      "loss": 0.536,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.7646220326423645,
+      "rewards/margins": 0.5907694101333618,
+      "rewards/rejected": -1.355391263961792,
       "step": 220
     },
     {
       "epoch": 0.48142333856619574,
-      "grad_norm": 30.893691078025817,
+      "grad_norm": 20.661508323720707,
       "learning_rate": 3.0893973387735683e-07,
-      "logits/chosen": -1.9850623607635498,
-      "logits/rejected": -1.8835790157318115,
-      "logps/chosen": -350.849853515625,
-      "logps/rejected": -432.809326171875,
-      "loss": 0.5227,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.0541720390319824,
-      "rewards/margins": 0.9999459385871887,
-      "rewards/rejected": -2.0541179180145264,
+      "logits/chosen": -2.536019802093506,
+      "logits/rejected": -2.50170636177063,
+      "logps/chosen": -335.34906005859375,
+      "logps/rejected": -395.7792053222656,
+      "loss": 0.5462,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.824562668800354,
+      "rewards/margins": 0.8087692260742188,
+      "rewards/rejected": -1.6333318948745728,
       "step": 230
     },
     {
       "epoch": 0.5023547880690737,
-      "grad_norm": 33.02799444972093,
+      "grad_norm": 25.41017506792381,
       "learning_rate": 2.910060778827554e-07,
-      "logits/chosen": -1.9802770614624023,
-      "logits/rejected": -1.880558967590332,
-      "logps/chosen": -384.36138916015625,
-      "logps/rejected": -446.0873107910156,
-      "loss": 0.4943,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.1203218698501587,
-      "rewards/margins": 0.8701934814453125,
-      "rewards/rejected": -1.9905153512954712,
+      "logits/chosen": -2.5316414833068848,
+      "logits/rejected": -2.4878182411193848,
+      "logps/chosen": -365.3668518066406,
+      "logps/rejected": -404.6741943359375,
+      "loss": 0.5097,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8398407697677612,
+      "rewards/margins": 0.6778165102005005,
+      "rewards/rejected": -1.5176571607589722,
       "step": 240
     },
     {
       "epoch": 0.5232862375719518,
-      "grad_norm": 35.534978981621464,
+      "grad_norm": 21.046624681616386,
       "learning_rate": 2.7285261601056697e-07,
-      "logits/chosen": -1.9159233570098877,
-      "logits/rejected": -1.778590202331543,
-      "logps/chosen": -370.39093017578125,
-      "logps/rejected": -430.15338134765625,
-      "loss": 0.5013,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.059476375579834,
-      "rewards/margins": 0.9878945350646973,
-      "rewards/rejected": -2.0473709106445312,
+      "logits/chosen": -2.5078229904174805,
+      "logits/rejected": -2.4572062492370605,
+      "logps/chosen": -362.0970153808594,
+      "logps/rejected": -402.87164306640625,
+      "loss": 0.5271,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.9026647806167603,
+      "rewards/margins": 0.8122144937515259,
+      "rewards/rejected": -1.7148792743682861,
       "step": 250
     },
     {
       "epoch": 0.54421768707483,
-      "grad_norm": 32.532057976810556,
+      "grad_norm": 26.951476961584774,
       "learning_rate": 2.5457665670441937e-07,
-      "logits/chosen": -2.016502857208252,
-      "logits/rejected": -2.0029537677764893,
-      "logps/chosen": -336.88287353515625,
-      "logps/rejected": -397.57806396484375,
-      "loss": 0.5046,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8760363459587097,
-      "rewards/margins": 0.7554494738578796,
-      "rewards/rejected": -1.631486177444458,
+      "logits/chosen": -2.545964479446411,
+      "logits/rejected": -2.563002586364746,
+      "logps/chosen": -341.8129577636719,
+      "logps/rejected": -384.24810791015625,
+      "loss": 0.5291,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8498622179031372,
+      "rewards/margins": 0.5883921384811401,
+      "rewards/rejected": -1.4382541179656982,
       "step": 260
     },
     {
       "epoch": 0.565149136577708,
-      "grad_norm": 29.326160875282234,
+      "grad_norm": 25.054855981780108,
       "learning_rate": 2.3627616503391812e-07,
-      "logits/chosen": -2.0074622631073,
-      "logits/rejected": -1.963415503501892,
-      "logps/chosen": -375.93359375,
-      "logps/rejected": -444.92010498046875,
-      "loss": 0.5195,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.9895309209823608,
-      "rewards/margins": 0.921257495880127,
-      "rewards/rejected": -1.9107885360717773,
+      "logits/chosen": -2.5025930404663086,
+      "logits/rejected": -2.4966609477996826,
+      "logps/chosen": -374.5225524902344,
+      "logps/rejected": -423.27667236328125,
+      "loss": 0.5365,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.8824374079704285,
+      "rewards/margins": 0.7567933797836304,
+      "rewards/rejected": -1.639230728149414,
       "step": 270
     },
     {
       "epoch": 0.5860805860805861,
-      "grad_norm": 31.08862280431646,
+      "grad_norm": 26.60953309509591,
       "learning_rate": 2.1804923757009882e-07,
-      "logits/chosen": -1.8468096256256104,
-      "logits/rejected": -1.7995507717132568,
-      "logps/chosen": -347.49713134765625,
-      "logps/rejected": -392.85498046875,
-      "loss": 0.5164,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.0229954719543457,
-      "rewards/margins": 0.7037075757980347,
-      "rewards/rejected": -1.7267030477523804,
+      "logits/chosen": -2.397477626800537,
+      "logits/rejected": -2.40055513381958,
+      "logps/chosen": -334.273681640625,
+      "logps/rejected": -364.7743835449219,
+      "loss": 0.5371,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8206404447555542,
+      "rewards/margins": 0.573003351688385,
+      "rewards/rejected": -1.3936437368392944,
       "step": 280
     },
     {
       "epoch": 0.6070120355834642,
-      "grad_norm": 35.420244394607494,
+      "grad_norm": 22.807917796831838,
       "learning_rate": 1.9999357655598891e-07,
-      "logits/chosen": -1.9150333404541016,
-      "logits/rejected": -1.819837212562561,
-      "logps/chosen": -338.6258544921875,
-      "logps/rejected": -407.5479431152344,
-      "loss": 0.5104,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.0374407768249512,
-      "rewards/margins": 0.7471441030502319,
-      "rewards/rejected": -1.7845849990844727,
+      "logits/chosen": -2.50712513923645,
+      "logits/rejected": -2.443718433380127,
+      "logps/chosen": -316.7132263183594,
+      "logps/rejected": -374.1191101074219,
+      "loss": 0.5309,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.7564879655838013,
+      "rewards/margins": 0.6396000385284424,
+      "rewards/rejected": -1.396087884902954,
       "step": 290
     },
     {
       "epoch": 0.6279434850863422,
-      "grad_norm": 29.946282422909608,
+      "grad_norm": 22.796315119157814,
       "learning_rate": 1.8220596619089573e-07,
-      "logits/chosen": -1.805354356765747,
-      "logits/rejected": -1.706356406211853,
-      "logps/chosen": -411.9443359375,
-      "logps/rejected": -455.328857421875,
-      "loss": 0.4834,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.102001667022705,
-      "rewards/margins": 0.8499504327774048,
-      "rewards/rejected": -1.9519522190093994,
+      "logits/chosen": -2.4191675186157227,
+      "logits/rejected": -2.383387804031372,
+      "logps/chosen": -396.5976257324219,
+      "logps/rejected": -420.3348693847656,
+      "loss": 0.5078,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.8598543405532837,
+      "rewards/margins": 0.6821789145469666,
+      "rewards/rejected": -1.5420334339141846,
       "step": 300
     },
     {
       "epoch": 0.6488749345892203,
-      "grad_norm": 33.48721686835146,
+      "grad_norm": 25.746010571486412,
       "learning_rate": 1.647817538357072e-07,
-      "logits/chosen": -1.8414630889892578,
-      "logits/rejected": -1.7567808628082275,
-      "logps/chosen": -416.7355041503906,
-      "logps/rejected": -473.2708435058594,
-      "loss": 0.4995,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.3046259880065918,
-      "rewards/margins": 1.056304931640625,
-      "rewards/rejected": -2.360930919647217,
+      "logits/chosen": -2.4751458168029785,
+      "logits/rejected": -2.4522910118103027,
+      "logps/chosen": -376.37713623046875,
+      "logps/rejected": -407.74664306640625,
+      "loss": 0.517,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.8200958371162415,
+      "rewards/margins": 0.8248319625854492,
+      "rewards/rejected": -1.644927978515625,
       "step": 310
     },
     {
       "epoch": 0.6698063840920984,
-      "grad_norm": 33.90432965296341,
+      "grad_norm": 23.866175857488113,
       "learning_rate": 1.478143389201113e-07,
-      "logits/chosen": -1.8603925704956055,
-      "logits/rejected": -1.7522039413452148,
-      "logps/chosen": -358.3260498046875,
-      "logps/rejected": -422.6536560058594,
-      "loss": 0.4826,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.1275660991668701,
-      "rewards/margins": 0.9410696029663086,
-      "rewards/rejected": -2.0686357021331787,
+      "logits/chosen": -2.4627318382263184,
+      "logits/rejected": -2.4139962196350098,
+      "logps/chosen": -325.8722229003906,
+      "logps/rejected": -377.2999572753906,
+      "loss": 0.4994,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.7399951815605164,
+      "rewards/margins": 0.8288515210151672,
+      "rewards/rejected": -1.5688468217849731,
       "step": 320
     },
     {
       "epoch": 0.6907378335949764,
-      "grad_norm": 27.513905071763368,
+      "grad_norm": 29.188939076017956,
       "learning_rate": 1.3139467229135998e-07,
-      "logits/chosen": -1.901578664779663,
-      "logits/rejected": -1.8754088878631592,
-      "logps/chosen": -363.4566345214844,
-      "logps/rejected": -448.67803955078125,
-      "loss": 0.4893,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.0841090679168701,
-      "rewards/margins": 0.8770705461502075,
-      "rewards/rejected": -1.9611797332763672,
+      "logits/chosen": -2.4468483924865723,
+      "logits/rejected": -2.4332222938537598,
+      "logps/chosen": -345.59136962890625,
+      "logps/rejected": -414.7674255371094,
+      "loss": 0.5127,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.8333412408828735,
+      "rewards/margins": 0.7380296587944031,
+      "rewards/rejected": -1.5713708400726318,
       "step": 330
     },
     {
       "epoch": 0.7116692830978545,
-      "grad_norm": 39.229104335542225,
+      "grad_norm": 32.23312179874695,
       "learning_rate": 1.1561076868822755e-07,
-      "logits/chosen": -1.7680823802947998,
-      "logits/rejected": -1.7113357782363892,
-      "logps/chosen": -401.6238708496094,
-      "logps/rejected": -475.1571350097656,
-      "loss": 0.4905,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.276782751083374,
-      "rewards/margins": 0.9465486407279968,
-      "rewards/rejected": -2.2233314514160156,
+      "logits/chosen": -2.373051166534424,
+      "logits/rejected": -2.356163501739502,
+      "logps/chosen": -370.69696044921875,
+      "logps/rejected": -425.781982421875,
+      "loss": 0.5113,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.8887789845466614,
+      "rewards/margins": 0.7785335779190063,
+      "rewards/rejected": -1.6673126220703125,
       "step": 340
     },
     {
       "epoch": 0.7326007326007326,
-      "grad_norm": 31.94987608162002,
+      "grad_norm": 26.011809781132502,
       "learning_rate": 1.0054723495346482e-07,
-      "logits/chosen": -1.806357979774475,
-      "logits/rejected": -1.7052780389785767,
-      "logps/chosen": -360.28607177734375,
-      "logps/rejected": -427.3250427246094,
-      "loss": 0.4608,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.143326759338379,
-      "rewards/margins": 0.9258495569229126,
-      "rewards/rejected": -2.069176197052002,
+      "logits/chosen": -2.46212100982666,
+      "logits/rejected": -2.4340288639068604,
+      "logps/chosen": -341.727783203125,
+      "logps/rejected": -391.04547119140625,
+      "loss": 0.4812,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8932917714118958,
+      "rewards/margins": 0.7555130124092102,
+      "rewards/rejected": -1.6488046646118164,
       "step": 350
     },
     {
       "epoch": 0.7535321821036107,
-      "grad_norm": 28.62799992820538,
+      "grad_norm": 23.13381761725731,
       "learning_rate": 8.628481651367875e-08,
-      "logits/chosen": -1.7983932495117188,
-      "logits/rejected": -1.6765844821929932,
-      "logps/chosen": -408.5673522949219,
-      "logps/rejected": -467.3724060058594,
-      "loss": 0.5179,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.1572364568710327,
-      "rewards/margins": 0.9978920817375183,
-      "rewards/rejected": -2.1551289558410645,
+      "logits/chosen": -2.46240496635437,
+      "logits/rejected": -2.4163765907287598,
+      "logps/chosen": -402.1018371582031,
+      "logps/rejected": -444.62384033203125,
+      "loss": 0.5344,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.0167683362960815,
+      "rewards/margins": 0.8525784611701965,
+      "rewards/rejected": -1.8693468570709229,
       "step": 360
     },
     {
       "epoch": 0.7744636316064888,
-      "grad_norm": 31.352439028541415,
+      "grad_norm": 23.213959633864068,
       "learning_rate": 7.289996455765748e-08,
-      "logits/chosen": -1.8380626440048218,
-      "logits/rejected": -1.7419719696044922,
-      "logps/chosen": -364.7669982910156,
-      "logps/rejected": -425.0555114746094,
-      "loss": 0.5005,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.1147412061691284,
-      "rewards/margins": 0.8857861757278442,
-      "rewards/rejected": -2.0005273818969727,
+      "logits/chosen": -2.461479663848877,
+      "logits/rejected": -2.42368483543396,
+      "logps/chosen": -361.43280029296875,
+      "logps/rejected": -407.6648864746094,
+      "loss": 0.5228,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.0166599750518799,
+      "rewards/margins": 0.7682315111160278,
+      "rewards/rejected": -1.7848914861679077,
       "step": 370
     },
     {
       "epoch": 0.7953950811093669,
-      "grad_norm": 26.28282670785366,
+      "grad_norm": 23.448752439895866,
       "learning_rate": 6.046442623320145e-08,
-      "logits/chosen": -1.6995561122894287,
-      "logits/rejected": -1.672106146812439,
-      "logps/chosen": -367.4508361816406,
-      "logps/rejected": -488.54913330078125,
-      "loss": 0.4978,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.3325788974761963,
-      "rewards/margins": 1.0493899583816528,
-      "rewards/rejected": -2.3819689750671387,
+      "logits/chosen": -2.3498904705047607,
+      "logits/rejected": -2.348081588745117,
+      "logps/chosen": -359.84429931640625,
+      "logps/rejected": -465.52752685546875,
+      "loss": 0.5171,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1946008205413818,
+      "rewards/margins": 0.8976942896842957,
+      "rewards/rejected": -2.0922951698303223,
       "step": 380
     },
     {
       "epoch": 0.8163265306122449,
-      "grad_norm": 29.780729258634107,
+      "grad_norm": 28.683016779162074,
       "learning_rate": 4.904486005914027e-08,
-      "logits/chosen": -1.8319686651229858,
-      "logits/rejected": -1.7405626773834229,
-      "logps/chosen": -448.5791931152344,
-      "logps/rejected": -498.88104248046875,
-      "loss": 0.4845,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.2597005367279053,
-      "rewards/margins": 0.8696678876876831,
-      "rewards/rejected": -2.129368543624878,
+      "logits/chosen": -2.4188392162323,
+      "logits/rejected": -2.3813109397888184,
+      "logps/chosen": -438.7664489746094,
+      "logps/rejected": -475.575439453125,
+      "loss": 0.5034,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0639350414276123,
+      "rewards/margins": 0.7629241943359375,
+      "rewards/rejected": -1.8268592357635498,
       "step": 390
     },
     {
       "epoch": 0.837257980115123,
-      "grad_norm": 30.094371408831872,
+      "grad_norm": 27.931926174751453,
       "learning_rate": 3.8702478614051345e-08,
-      "logits/chosen": -1.7541265487670898,
-      "logits/rejected": -1.6716575622558594,
-      "logps/chosen": -365.5286560058594,
-      "logps/rejected": -440.4735412597656,
-      "loss": 0.5052,
+      "logits/chosen": -2.392533779144287,
+      "logits/rejected": -2.3728108406066895,
+      "logps/chosen": -349.75946044921875,
+      "logps/rejected": -409.954833984375,
+      "loss": 0.523,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.2250964641571045,
-      "rewards/margins": 0.877382755279541,
-      "rewards/rejected": -2.1024792194366455,
+      "rewards/chosen": -1.0062363147735596,
+      "rewards/margins": 0.7513880133628845,
+      "rewards/rejected": -1.7576243877410889,
       "step": 400
     },
     {
       "epoch": 0.858189429618001,
-      "grad_norm": 32.623663420886714,
+      "grad_norm": 22.79792813855284,
       "learning_rate": 2.9492720416985e-08,
-      "logits/chosen": -1.8887017965316772,
-      "logits/rejected": -1.79939866065979,
-      "logps/chosen": -404.93719482421875,
-      "logps/rejected": -470.50714111328125,
-      "loss": 0.4989,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.173086404800415,
-      "rewards/margins": 0.9763132333755493,
-      "rewards/rejected": -2.149399518966675,
+      "logits/chosen": -2.470493793487549,
+      "logits/rejected": -2.4561924934387207,
+      "logps/chosen": -392.3372802734375,
+      "logps/rejected": -443.60693359375,
+      "loss": 0.5171,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9740950465202332,
+      "rewards/margins": 0.8542107343673706,
+      "rewards/rejected": -1.8283056020736694,
       "step": 410
     },
     {
       "epoch": 0.8791208791208791,
-      "grad_norm": 32.9180383847494,
+      "grad_norm": 29.28251834801288,
       "learning_rate": 2.1464952759020856e-08,
-      "logits/chosen": -1.7282394170761108,
-      "logits/rejected": -1.6893911361694336,
-      "logps/chosen": -366.7571716308594,
-      "logps/rejected": -468.3814392089844,
-      "loss": 0.4868,
+      "logits/chosen": -2.359532356262207,
+      "logits/rejected": -2.3633604049682617,
+      "logps/chosen": -354.50079345703125,
+      "logps/rejected": -437.41461181640625,
+      "loss": 0.5054,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2342027425765991,
-      "rewards/margins": 0.9661720991134644,
-      "rewards/rejected": -2.2003750801086426,
+      "rewards/chosen": -1.0463224649429321,
+      "rewards/margins": 0.7997694611549377,
+      "rewards/rejected": -1.8460919857025146,
       "step": 420
     },
     {
       "epoch": 0.9000523286237572,
-      "grad_norm": 33.302644642921706,
+      "grad_norm": 25.430890882929,
       "learning_rate": 1.4662207078575684e-08,
-      "logits/chosen": -1.7425053119659424,
-      "logits/rejected": -1.6233768463134766,
-      "logps/chosen": -397.09466552734375,
-      "logps/rejected": -470.15692138671875,
-      "loss": 0.4755,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.2036662101745605,
-      "rewards/margins": 0.977012038230896,
-      "rewards/rejected": -2.180678129196167,
+      "logits/chosen": -2.3850908279418945,
+      "logits/rejected": -2.3233494758605957,
+      "logps/chosen": -382.2372131347656,
+      "logps/rejected": -434.57659912109375,
+      "loss": 0.4949,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.9799245595932007,
+      "rewards/margins": 0.7894011735916138,
+      "rewards/rejected": -1.769325613975525,
       "step": 430
     },
     {
       "epoch": 0.9209837781266352,
-      "grad_norm": 30.223398646627228,
+      "grad_norm": 25.594864644940884,
       "learning_rate": 9.12094829893642e-09,
-      "logits/chosen": -1.86935555934906,
-      "logits/rejected": -1.7585684061050415,
-      "logps/chosen": -369.0350341796875,
-      "logps/rejected": -429.52020263671875,
-      "loss": 0.4945,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.1897361278533936,
-      "rewards/margins": 0.9693399667739868,
-      "rewards/rejected": -2.15907621383667,
+      "logits/chosen": -2.4849045276641846,
+      "logits/rejected": -2.4323153495788574,
+      "logps/chosen": -349.52801513671875,
+      "logps/rejected": -390.45703125,
+      "loss": 0.5063,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9329977035522461,
+      "rewards/margins": 0.772846519947052,
+      "rewards/rejected": -1.7058441638946533,
       "step": 440
     },
     {
       "epoch": 0.9419152276295133,
-      "grad_norm": 32.3520965832113,
+      "grad_norm": 26.32867623084858,
       "learning_rate": 4.8708793644441086e-09,
-      "logits/chosen": -1.600304365158081,
-      "logits/rejected": -1.5472590923309326,
-      "logps/chosen": -384.4742431640625,
-      "logps/rejected": -480.8585510253906,
-      "loss": 0.4854,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.2649081945419312,
-      "rewards/margins": 1.0646650791168213,
-      "rewards/rejected": -2.329573154449463,
+      "logits/chosen": -2.244398593902588,
+      "logits/rejected": -2.2427639961242676,
+      "logps/chosen": -362.46478271484375,
+      "logps/rejected": -438.66748046875,
+      "loss": 0.5081,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.979316234588623,
+      "rewards/margins": 0.872042179107666,
+      "rewards/rejected": -1.85135817527771,
       "step": 450
     },
     {
       "epoch": 0.9628466771323915,
-      "grad_norm": 40.57863055733759,
+      "grad_norm": 41.14956290790987,
       "learning_rate": 1.9347820230782295e-09,
-      "logits/chosen": -1.7524124383926392,
-      "logits/rejected": -1.6150585412979126,
-      "logps/chosen": -385.82073974609375,
-      "logps/rejected": -439.78973388671875,
-      "loss": 0.4974,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.265455722808838,
-      "rewards/margins": 0.9535617828369141,
-      "rewards/rejected": -2.219017505645752,
+      "logits/chosen": -2.3787941932678223,
+      "logits/rejected": -2.329960346221924,
+      "logps/chosen": -364.7981872558594,
+      "logps/rejected": -403.34716796875,
+      "loss": 0.5164,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.983986496925354,
+      "rewards/margins": 0.8341706395149231,
+      "rewards/rejected": -1.8181571960449219,
       "step": 460
     },
     {
       "epoch": 0.9837781266352695,
-      "grad_norm": 26.449373393329658,
+      "grad_norm": 22.500477554148656,
       "learning_rate": 3.2839470889836627e-10,
-      "logits/chosen": -1.7558813095092773,
-      "logits/rejected": -1.6784464120864868,
-      "logps/chosen": -399.96173095703125,
-      "logps/rejected": -463.38787841796875,
-      "loss": 0.4838,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.169880747795105,
-      "rewards/margins": 0.9056078195571899,
-      "rewards/rejected": -2.075488567352295,
+      "logits/chosen": -2.385188579559326,
+      "logits/rejected": -2.3625881671905518,
+      "logps/chosen": -382.80291748046875,
+      "logps/rejected": -424.16400146484375,
+      "loss": 0.4975,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9208614230155945,
+      "rewards/margins": 0.7104892134666443,
+      "rewards/rejected": -1.6313507556915283,
       "step": 470
     },
     {
       "epoch": 0.9984301412872841,
-      "eval_logits/chosen": -1.7399183511734009,
-      "eval_logits/rejected": -1.6741559505462646,
-      "eval_logps/chosen": -363.3537902832031,
-      "eval_logps/rejected": -469.47021484375,
-      "eval_loss": 0.49291545152664185,
-      "eval_rewards/accuracies": 0.78515625,
-      "eval_rewards/chosen": -1.1416336297988892,
-      "eval_rewards/margins": 1.0502383708953857,
-      "eval_rewards/rejected": -2.1918721199035645,
-      "eval_runtime": 168.8714,
-      "eval_samples_per_second": 11.843,
-      "eval_steps_per_second": 0.189,
+      "eval_logits/chosen": -2.4335575103759766,
+      "eval_logits/rejected": -2.4131321907043457,
+      "eval_logps/chosen": -346.3243713378906,
+      "eval_logps/rejected": -432.3867492675781,
+      "eval_loss": 0.5131625533103943,
+      "eval_rewards/accuracies": 0.78125,
+      "eval_rewards/chosen": -0.89111328125,
+      "eval_rewards/margins": 0.8651110529899597,
+      "eval_rewards/rejected": -1.756224274635315,
+      "eval_runtime": 168.71,
+      "eval_samples_per_second": 11.855,
+      "eval_steps_per_second": 0.19,
       "step": 477
     },
     {
       "epoch": 0.9984301412872841,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.5343590847351266,
-      "train_runtime": 13524.9429,
-      "train_samples_per_second": 4.52,
-      "train_steps_per_second": 0.035
+      "train_loss": 0.5516081291174738,
+      "train_runtime": 26908.0955,
+      "train_samples_per_second": 2.272,
+      "train_steps_per_second": 0.018
     }
   ],
   "logging_steps": 10,