Model save

README.md

@@ -16,15 +16,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5765
-- Rewards/chosen: -0.2836
-- Rewards/rejected: -0.7661
-- Rewards/accuracies: 0.75
-- Rewards/margins: 0.4825
-- Logps/rejected: -342.4585
-- Logps/chosen: -295.9262
-- Logits/rejected: -2.6851
-- Logits/chosen: -2.7117
+- Loss: 0.5669
+- Rewards/chosen: -0.4034
+- Rewards/rejected: -0.9628
+- Rewards/accuracies: 0.7461
+- Rewards/margins: 0.5594
+- Logps/rejected: -372.7966
+- Logps/chosen: -315.5382
+- Logits/rejected: -2.3014
+- Logits/chosen: -2.3480
 
 ## Model description
 
@@ -61,7 +61,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5798        | 0.4982 | 238  | 0.5765          | -0.2836        | -0.7661          | 0.75               | 0.4825          | -342.4585      | -295.9262    | -2.6851         | -2.7117       |
+| 0.5667        | 0.4982 | 238  | 0.5669          | -0.4034        | -0.9628          | 0.7461             | 0.5594          | -372.7966      | -315.5382    | -2.3014         | -2.3480       |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.4981684981684982,
     "total_flos": 0.0,
-    "train_loss": 0.6047679316095945,
-    "train_runtime": 7201.8834,
+    "train_loss": 0.6061525304778284,
+    "train_runtime": 7204.502,
     "train_samples": 61134,
-    "train_samples_per_second": 4.23,
+    "train_samples_per_second": 4.228,
     "train_steps_per_second": 0.033
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "data/sft/zephyr-7b-sft-1e-every25/checkpoint-900",
+  "_name_or_path": "data/sft/zephyr-7b-sft-1e-every25/checkpoint-600",
   "architectures": [
     "MistralForCausalLM"
   ],

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d93eb6300b877466ea2f4e521cf819b0ef638a9ea09ad57832ad62ea8917b3f
+oid sha256:e4d2c8e646e5701192e73de8c38e6d4056e9f571ca8e10a64d702d40172ec86e
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09174fa607bcca81805e06045dda478fdf2a5dc93bfb0a52c322bf4fa96de23a
+oid sha256:7050cc11c07f527e41773fffa439f1cabdeb7522827afd0523198340ec011529
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81fa5d1748aec6d9e1442511af340a774590249d11e7d8f0b2e7d2f38c73ea44
+oid sha256:8b3a7c01323eff3dc3201152846f798738d11befcbd65f5366c003220d7aa2f6
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.4981684981684982,
     "total_flos": 0.0,
-    "train_loss": 0.6047679316095945,
-    "train_runtime": 7201.8834,
+    "train_loss": 0.6061525304778284,
+    "train_runtime": 7204.502,
     "train_samples": 61134,
-    "train_samples_per_second": 4.23,
+    "train_samples_per_second": 4.228,
     "train_steps_per_second": 0.033
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0020931449502878076,
-      "grad_norm": 12.610560260880211,
+      "grad_norm": 26.601607092770625,
       "learning_rate": 2.083333333333333e-08,
-      "logits/chosen": -2.99812388420105,
-      "logits/rejected": -2.9638350009918213,
-      "logps/chosen": -315.4977722167969,
-      "logps/rejected": -361.9093322753906,
+      "logits/chosen": -2.9139022827148438,
+      "logits/rejected": -2.8787596225738525,
+      "logps/chosen": -325.8533020019531,
+      "logps/rejected": -372.9187927246094,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,372 +25,372 @@
     },
     {
       "epoch": 0.020931449502878074,
-      "grad_norm": 9.936649993419687,
+      "grad_norm": 16.144681025948177,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.788989543914795,
-      "logits/rejected": -2.7662315368652344,
-      "logps/chosen": -314.0841064453125,
-      "logps/rejected": -282.2810974121094,
-      "loss": 0.693,
-      "rewards/accuracies": 0.4722222089767456,
-      "rewards/chosen": 0.0008651986136101186,
-      "rewards/margins": 0.0003082120092585683,
-      "rewards/rejected": 0.0005569865461438894,
+      "logits/chosen": -2.730161428451538,
+      "logits/rejected": -2.7064931392669678,
+      "logps/chosen": -331.04364013671875,
+      "logps/rejected": -309.8411865234375,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": 0.001100256573408842,
+      "rewards/margins": 0.0007432710262946784,
+      "rewards/rejected": 0.0003569853724911809,
       "step": 10
     },
     {
       "epoch": 0.04186289900575615,
-      "grad_norm": 8.667356504170359,
+      "grad_norm": 14.575580508073863,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.8659820556640625,
-      "logits/rejected": -2.8171162605285645,
-      "logps/chosen": -311.6136779785156,
-      "logps/rejected": -284.09893798828125,
-      "loss": 0.6909,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.013467146083712578,
-      "rewards/margins": 0.004726298153400421,
-      "rewards/rejected": 0.008740848861634731,
+      "logits/chosen": -2.798081874847412,
+      "logits/rejected": -2.7473387718200684,
+      "logps/chosen": -321.3909912109375,
+      "logps/rejected": -299.4921569824219,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.006536015775054693,
+      "rewards/margins": 0.0068586282432079315,
+      "rewards/rejected": -0.0003226128756068647,
       "step": 20
     },
     {
       "epoch": 0.06279434850863422,
-      "grad_norm": 8.034528876736365,
+      "grad_norm": 8.070909879894359,
       "learning_rate": 4.99030821197584e-07,
-      "logits/chosen": -2.8421683311462402,
-      "logits/rejected": -2.7854294776916504,
-      "logps/chosen": -289.38385009765625,
-      "logps/rejected": -247.28732299804688,
-      "loss": 0.6833,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.05833645910024643,
-      "rewards/margins": 0.024186396971344948,
-      "rewards/rejected": 0.03415006399154663,
+      "logits/chosen": -2.7842161655426025,
+      "logits/rejected": -2.728454113006592,
+      "logps/chosen": -304.2439880371094,
+      "logps/rejected": -275.6659851074219,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.04330765828490257,
+      "rewards/margins": 0.05840452387928963,
+      "rewards/rejected": -0.015096860937774181,
       "step": 30
     },
     {
       "epoch": 0.0837257980115123,
-      "grad_norm": 7.833193286645188,
+      "grad_norm": 8.530980057951403,
       "learning_rate": 4.931352528237397e-07,
-      "logits/chosen": -2.7391459941864014,
-      "logits/rejected": -2.716399669647217,
-      "logps/chosen": -269.075927734375,
-      "logps/rejected": -262.9975891113281,
-      "loss": 0.6682,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.052591562271118164,
-      "rewards/margins": 0.0480208583176136,
-      "rewards/rejected": 0.0045707011595368385,
+      "logits/chosen": -2.7036166191101074,
+      "logits/rejected": -2.671600103378296,
+      "logps/chosen": -289.6343078613281,
+      "logps/rejected": -277.79425048828125,
+      "loss": 0.6767,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.02837621606886387,
+      "rewards/margins": 0.037141989916563034,
+      "rewards/rejected": -0.008765773847699165,
       "step": 40
     },
     {
       "epoch": 0.10465724751439037,
-      "grad_norm": 8.72813708365478,
+      "grad_norm": 11.721866187252868,
       "learning_rate": 4.820092227512735e-07,
-      "logits/chosen": -2.7570505142211914,
-      "logits/rejected": -2.696572780609131,
-      "logps/chosen": -282.76092529296875,
-      "logps/rejected": -286.132568359375,
-      "loss": 0.6548,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.010979737155139446,
-      "rewards/margins": 0.09300607442855835,
-      "rewards/rejected": -0.10398580878973007,
+      "logits/chosen": -2.740696430206299,
+      "logits/rejected": -2.6848952770233154,
+      "logps/chosen": -290.8323059082031,
+      "logps/rejected": -310.22576904296875,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.014719474129378796,
+      "rewards/margins": 0.10523072630167007,
+      "rewards/rejected": -0.11995019763708115,
       "step": 50
     },
     {
       "epoch": 0.12558869701726844,
-      "grad_norm": 13.480613892862241,
+      "grad_norm": 10.915648900872057,
       "learning_rate": 4.658920803689553e-07,
-      "logits/chosen": -2.789520502090454,
-      "logits/rejected": -2.7424087524414062,
-      "logps/chosen": -258.74908447265625,
-      "logps/rejected": -264.25225830078125,
-      "loss": 0.6357,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.012783573940396309,
-      "rewards/margins": 0.15775156021118164,
-      "rewards/rejected": -0.1705351322889328,
+      "logits/chosen": -2.785388469696045,
+      "logits/rejected": -2.744814395904541,
+      "logps/chosen": -268.2928466796875,
+      "logps/rejected": -272.32122802734375,
+      "loss": 0.6461,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.02511899545788765,
+      "rewards/margins": 0.14482316374778748,
+      "rewards/rejected": -0.16994217038154602,
       "step": 60
     },
     {
       "epoch": 0.14652014652014653,
-      "grad_norm": 10.234102018070638,
+      "grad_norm": 11.152537838170248,
       "learning_rate": 4.4513054666826144e-07,
-      "logits/chosen": -2.7376105785369873,
-      "logits/rejected": -2.709526300430298,
-      "logps/chosen": -311.8622131347656,
-      "logps/rejected": -316.22576904296875,
-      "loss": 0.6163,
+      "logits/chosen": -2.745926856994629,
+      "logits/rejected": -2.7201263904571533,
+      "logps/chosen": -332.7641296386719,
+      "logps/rejected": -326.02862548828125,
+      "loss": 0.6338,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.11481525003910065,
-      "rewards/margins": 0.20760241150856018,
-      "rewards/rejected": -0.32241764664649963,
+      "rewards/chosen": -0.12678995728492737,
+      "rewards/margins": 0.15600599348545074,
+      "rewards/rejected": -0.2827959656715393,
       "step": 70
     },
     {
       "epoch": 0.1674515960230246,
-      "grad_norm": 20.552548250652773,
+      "grad_norm": 20.747423551686875,
       "learning_rate": 4.201712553872657e-07,
-      "logits/chosen": -2.816467761993408,
-      "logits/rejected": -2.7381579875946045,
-      "logps/chosen": -341.7183837890625,
-      "logps/rejected": -293.2793884277344,
-      "loss": 0.6055,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.03545045107603073,
-      "rewards/margins": 0.28459519147872925,
-      "rewards/rejected": -0.3200456500053406,
+      "logits/chosen": -2.8221614360809326,
+      "logits/rejected": -2.75636887550354,
+      "logps/chosen": -364.9082336425781,
+      "logps/rejected": -309.4717102050781,
+      "loss": 0.6129,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.07667700946331024,
+      "rewards/margins": 0.23987922072410583,
+      "rewards/rejected": -0.3165562152862549,
       "step": 80
     },
     {
       "epoch": 0.18838304552590268,
-      "grad_norm": 13.720188030623374,
+      "grad_norm": 12.643373189764793,
       "learning_rate": 3.9155114477557926e-07,
-      "logits/chosen": -2.7560811042785645,
-      "logits/rejected": -2.7106573581695557,
-      "logps/chosen": -275.1942443847656,
-      "logps/rejected": -306.68206787109375,
-      "loss": 0.5887,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.1888897567987442,
-      "rewards/margins": 0.30051952600479126,
-      "rewards/rejected": -0.48940929770469666,
+      "logits/chosen": -2.7849178314208984,
+      "logits/rejected": -2.748291492462158,
+      "logps/chosen": -287.956787109375,
+      "logps/rejected": -340.713134765625,
+      "loss": 0.5971,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.24098113179206848,
+      "rewards/margins": 0.3181685507297516,
+      "rewards/rejected": -0.5591496825218201,
       "step": 90
     },
     {
       "epoch": 0.20931449502878074,
-      "grad_norm": 16.303248781308948,
+      "grad_norm": 13.623537820936637,
       "learning_rate": 3.598859066780754e-07,
-      "logits/chosen": -2.7324023246765137,
-      "logits/rejected": -2.7062618732452393,
-      "logps/chosen": -322.99603271484375,
-      "logps/rejected": -339.0654602050781,
-      "loss": 0.5902,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.19065451622009277,
-      "rewards/margins": 0.39800626039505005,
-      "rewards/rejected": -0.588660717010498,
+      "logits/chosen": -2.76792311668396,
+      "logits/rejected": -2.7459118366241455,
+      "logps/chosen": -346.0640563964844,
+      "logps/rejected": -348.5198059082031,
+      "loss": 0.6098,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.2652357220649719,
+      "rewards/margins": 0.34126365184783936,
+      "rewards/rejected": -0.6064993739128113,
       "step": 100
     },
     {
       "epoch": 0.2302459445316588,
-      "grad_norm": 20.16983103502839,
+      "grad_norm": 15.41884704737969,
       "learning_rate": 3.2585674142717477e-07,
-      "logits/chosen": -2.7619636058807373,
-      "logits/rejected": -2.713339328765869,
-      "logps/chosen": -338.515869140625,
-      "logps/rejected": -351.4220886230469,
-      "loss": 0.5967,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.4326956868171692,
-      "rewards/margins": 0.302705854177475,
-      "rewards/rejected": -0.7354015111923218,
+      "logits/chosen": -2.7856485843658447,
+      "logits/rejected": -2.7417104244232178,
+      "logps/chosen": -335.01116943359375,
+      "logps/rejected": -349.71392822265625,
+      "loss": 0.6003,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.32472461462020874,
+      "rewards/margins": 0.3165324330329895,
+      "rewards/rejected": -0.6412570476531982,
       "step": 110
     },
     {
       "epoch": 0.25117739403453687,
-      "grad_norm": 18.288819495100896,
+      "grad_norm": 15.439444003212843,
       "learning_rate": 2.9019570347986706e-07,
-      "logits/chosen": -2.670757293701172,
-      "logits/rejected": -2.6768569946289062,
-      "logps/chosen": -309.2828674316406,
-      "logps/rejected": -339.7432861328125,
-      "loss": 0.579,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.33222970366477966,
-      "rewards/margins": 0.40109533071517944,
-      "rewards/rejected": -0.7333250045776367,
+      "logits/chosen": -2.710737705230713,
+      "logits/rejected": -2.718116283416748,
+      "logps/chosen": -330.0270690917969,
+      "logps/rejected": -354.4695739746094,
+      "loss": 0.5832,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.3180462718009949,
+      "rewards/margins": 0.3863913416862488,
+      "rewards/rejected": -0.7044375538825989,
       "step": 120
     },
     {
       "epoch": 0.272108843537415,
-      "grad_norm": 15.354064982647847,
+      "grad_norm": 42.02069053016682,
       "learning_rate": 2.536699530523291e-07,
-      "logits/chosen": -2.722832679748535,
-      "logits/rejected": -2.7066872119903564,
-      "logps/chosen": -318.85833740234375,
-      "logps/rejected": -335.77911376953125,
-      "loss": 0.5666,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.2824149429798126,
-      "rewards/margins": 0.33272355794906616,
-      "rewards/rejected": -0.6151384711265564,
+      "logits/chosen": -2.7691874504089355,
+      "logits/rejected": -2.752965211868286,
+      "logps/chosen": -347.0602722167969,
+      "logps/rejected": -357.85504150390625,
+      "loss": 0.5817,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.2763240337371826,
+      "rewards/margins": 0.27957138419151306,
+      "rewards/rejected": -0.5558954477310181,
       "step": 130
     },
     {
       "epoch": 0.29304029304029305,
-      "grad_norm": 17.28336393009731,
+      "grad_norm": 24.79876226607902,
       "learning_rate": 2.1706525253979534e-07,
-      "logits/chosen": -2.7583320140838623,
-      "logits/rejected": -2.729793071746826,
-      "logps/chosen": -346.2691955566406,
-      "logps/rejected": -344.5732116699219,
-      "loss": 0.5763,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.24767926335334778,
-      "rewards/margins": 0.4790892004966736,
-      "rewards/rejected": -0.7267683744430542,
+      "logits/chosen": -2.76993989944458,
+      "logits/rejected": -2.750406265258789,
+      "logps/chosen": -360.31005859375,
+      "logps/rejected": -362.851806640625,
+      "loss": 0.588,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.26679104566574097,
+      "rewards/margins": 0.46773427724838257,
+      "rewards/rejected": -0.7345253229141235,
       "step": 140
     },
     {
       "epoch": 0.3139717425431711,
-      "grad_norm": 19.361596281568826,
+      "grad_norm": 22.26447349378322,
       "learning_rate": 1.8116906275593507e-07,
-      "logits/chosen": -2.7468533515930176,
-      "logits/rejected": -2.7160446643829346,
-      "logps/chosen": -323.2176208496094,
-      "logps/rejected": -324.5989990234375,
-      "loss": 0.5796,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.3653566539287567,
-      "rewards/margins": 0.3610517382621765,
-      "rewards/rejected": -0.7264083623886108,
+      "logits/chosen": -2.7364563941955566,
+      "logits/rejected": -2.7079169750213623,
+      "logps/chosen": -351.19189453125,
+      "logps/rejected": -349.318603515625,
+      "loss": 0.5755,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.43332210183143616,
+      "rewards/margins": 0.3800078332424164,
+      "rewards/rejected": -0.8133300542831421,
       "step": 150
     },
     {
       "epoch": 0.3349031920460492,
-      "grad_norm": 16.313995999292402,
+      "grad_norm": 26.67231794642688,
       "learning_rate": 1.4675360263490295e-07,
-      "logits/chosen": -2.7838118076324463,
-      "logits/rejected": -2.7365283966064453,
-      "logps/chosen": -319.6917724609375,
-      "logps/rejected": -340.581787109375,
-      "loss": 0.5759,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.40762004256248474,
-      "rewards/margins": 0.4462064802646637,
-      "rewards/rejected": -0.8538265228271484,
+      "logits/chosen": -2.730668544769287,
+      "logits/rejected": -2.683061122894287,
+      "logps/chosen": -329.1626281738281,
+      "logps/rejected": -350.14288330078125,
+      "loss": 0.5765,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4029548764228821,
+      "rewards/margins": 0.5021006464958191,
+      "rewards/rejected": -0.9050555229187012,
       "step": 160
     },
     {
       "epoch": 0.35583464154892724,
-      "grad_norm": 22.925451744364146,
+      "grad_norm": 20.223571694588983,
       "learning_rate": 1.1455923682523475e-07,
-      "logits/chosen": -2.659465789794922,
-      "logits/rejected": -2.644275188446045,
-      "logps/chosen": -311.614990234375,
-      "logps/rejected": -323.3099670410156,
-      "loss": 0.585,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.4685409665107727,
-      "rewards/margins": 0.36185160279273987,
-      "rewards/rejected": -0.8303925395011902,
+      "logits/chosen": -2.532707691192627,
+      "logits/rejected": -2.512026309967041,
+      "logps/chosen": -324.9989929199219,
+      "logps/rejected": -334.66888427734375,
+      "loss": 0.5755,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.36262771487236023,
+      "rewards/margins": 0.39737457036972046,
+      "rewards/rejected": -0.7600023150444031,
       "step": 170
     },
     {
       "epoch": 0.37676609105180536,
-      "grad_norm": 13.975706307085083,
+      "grad_norm": 18.84156651008044,
       "learning_rate": 8.527854855097224e-08,
-      "logits/chosen": -2.7547802925109863,
-      "logits/rejected": -2.7258901596069336,
-      "logps/chosen": -313.0289001464844,
-      "logps/rejected": -321.9787902832031,
-      "loss": 0.5538,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.37977224588394165,
-      "rewards/margins": 0.3600301146507263,
-      "rewards/rejected": -0.739802360534668,
+      "logits/chosen": -2.5160155296325684,
+      "logits/rejected": -2.4737634658813477,
+      "logps/chosen": -328.61224365234375,
+      "logps/rejected": -344.9612121582031,
+      "loss": 0.5499,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.454254150390625,
+      "rewards/margins": 0.40306931734085083,
+      "rewards/rejected": -0.8573234677314758,
       "step": 180
     },
     {
       "epoch": 0.3976975405546834,
-      "grad_norm": 16.933160186335314,
+      "grad_norm": 32.06520543659375,
       "learning_rate": 5.9541440373546445e-08,
-      "logits/chosen": -2.689134120941162,
-      "logits/rejected": -2.664074420928955,
-      "logps/chosen": -296.32672119140625,
-      "logps/rejected": -319.95001220703125,
-      "loss": 0.5755,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.34621429443359375,
-      "rewards/margins": 0.2947639226913452,
-      "rewards/rejected": -0.6409782767295837,
+      "logits/chosen": -2.3760242462158203,
+      "logits/rejected": -2.352184772491455,
+      "logps/chosen": -323.0282897949219,
+      "logps/rejected": -362.2243957519531,
+      "loss": 0.5613,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5423271059989929,
+      "rewards/margins": 0.3743038773536682,
+      "rewards/rejected": -0.9166310429573059,
       "step": 190
     },
     {
       "epoch": 0.4186289900575615,
-      "grad_norm": 21.908273233954745,
+      "grad_norm": 23.15044859855663,
       "learning_rate": 3.790158337517127e-08,
-      "logits/chosen": -2.653926372528076,
-      "logits/rejected": -2.6524300575256348,
-      "logps/chosen": -289.92926025390625,
-      "logps/rejected": -341.435546875,
-      "loss": 0.5823,
+      "logits/chosen": -2.311206817626953,
+      "logits/rejected": -2.2949588298797607,
+      "logps/chosen": -342.80731201171875,
+      "logps/rejected": -394.5885314941406,
+      "loss": 0.5711,
       "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.3269258737564087,
-      "rewards/margins": 0.3465423882007599,
-      "rewards/rejected": -0.6734683513641357,
+      "rewards/chosen": -0.5416313409805298,
+      "rewards/margins": 0.41420310735702515,
+      "rewards/rejected": -0.9558345079421997,
       "step": 200
     },
     {
       "epoch": 0.43956043956043955,
-      "grad_norm": 18.994509414379117,
+      "grad_norm": 19.06533234677666,
       "learning_rate": 2.0824506276503894e-08,
-      "logits/chosen": -2.7132248878479004,
-      "logits/rejected": -2.648906946182251,
-      "logps/chosen": -352.23211669921875,
-      "logps/rejected": -327.8628234863281,
-      "loss": 0.5851,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.38004761934280396,
-      "rewards/margins": 0.3670490086078644,
-      "rewards/rejected": -0.7470966577529907,
+      "logits/chosen": -2.3925139904022217,
+      "logits/rejected": -2.268831491470337,
+      "logps/chosen": -395.4692077636719,
+      "logps/rejected": -360.0453186035156,
+      "loss": 0.5805,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.5327693223953247,
+      "rewards/margins": 0.4689061641693115,
+      "rewards/rejected": -1.0016754865646362,
       "step": 210
     },
     {
       "epoch": 0.4604918890633176,
-      "grad_norm": 16.241884556359377,
+      "grad_norm": 24.03298915942009,
       "learning_rate": 8.677580722139671e-09,
-      "logits/chosen": -2.635542154312134,
-      "logits/rejected": -2.6469016075134277,
-      "logps/chosen": -288.9513244628906,
-      "logps/rejected": -323.00494384765625,
-      "loss": 0.573,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.3638390898704529,
-      "rewards/margins": 0.3917424976825714,
-      "rewards/rejected": -0.7555815577507019,
+      "logits/chosen": -2.316939353942871,
+      "logits/rejected": -2.2927820682525635,
+      "logps/chosen": -342.931640625,
+      "logps/rejected": -364.62982177734375,
+      "loss": 0.5674,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.501736581325531,
+      "rewards/margins": 0.49570217728614807,
+      "rewards/rejected": -0.9974387884140015,
       "step": 220
     },
     {
       "epoch": 0.48142333856619574,
-      "grad_norm": 22.899672406649472,
+      "grad_norm": 19.987346780573713,
       "learning_rate": 1.722118176089915e-09,
-      "logits/chosen": -2.742950439453125,
-      "logits/rejected": -2.702911376953125,
-      "logps/chosen": -292.0196533203125,
-      "logps/rejected": -330.7127380371094,
-      "loss": 0.5798,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.25979113578796387,
-      "rewards/margins": 0.5986371040344238,
-      "rewards/rejected": -0.8584282994270325,
+      "logits/chosen": -2.4182724952697754,
+      "logits/rejected": -2.337088108062744,
+      "logps/chosen": -337.9970703125,
+      "logps/rejected": -381.84320068359375,
+      "loss": 0.5667,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.39490121603012085,
+      "rewards/margins": 0.7370277643203735,
+      "rewards/rejected": -1.1319290399551392,
       "step": 230
     },
     {
       "epoch": 0.4981684981684982,
-      "eval_logits/chosen": -2.711683750152588,
-      "eval_logits/rejected": -2.68514084815979,
-      "eval_logps/chosen": -295.9261779785156,
-      "eval_logps/rejected": -342.4585266113281,
-      "eval_loss": 0.576471209526062,
-      "eval_rewards/accuracies": 0.75,
-      "eval_rewards/chosen": -0.283584326505661,
-      "eval_rewards/margins": 0.48247623443603516,
-      "eval_rewards/rejected": -0.7660605311393738,
-      "eval_runtime": 168.4238,
-      "eval_samples_per_second": 11.875,
-      "eval_steps_per_second": 0.19,
+      "eval_logits/chosen": -2.3480114936828613,
+      "eval_logits/rejected": -2.3013908863067627,
+      "eval_logps/chosen": -315.5382385253906,
+      "eval_logps/rejected": -372.7966003417969,
+      "eval_loss": 0.5668980479240417,
+      "eval_rewards/accuracies": 0.74609375,
+      "eval_rewards/chosen": -0.4034212827682495,
+      "eval_rewards/margins": 0.5594114065170288,
+      "eval_rewards/rejected": -0.9628326892852783,
+      "eval_runtime": 171.8973,
+      "eval_samples_per_second": 11.635,
+      "eval_steps_per_second": 0.186,
       "step": 238
     },
     {
       "epoch": 0.4981684981684982,
       "step": 238,
       "total_flos": 0.0,
-      "train_loss": 0.6047679316095945,
-      "train_runtime": 7201.8834,
-      "train_samples_per_second": 4.23,
+      "train_loss": 0.6061525304778284,
+      "train_runtime": 7204.502,
+      "train_samples_per_second": 4.228,
       "train_steps_per_second": 0.033
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:13885dc2cfddf0eb7080d5bad9f264e574a65e3f8468c00bc6a65382edb01768
+oid sha256:a917c64a8bc731aa9c248cac3f53adb5e79283709a6de18e8be835bb3be98d41
 size 7672