Model save

README.md

@@ -16,15 +16,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5754
-- Rewards/chosen: -0.2750
-- Rewards/rejected: -0.7430
+- Loss: 0.5765
+- Rewards/chosen: -0.2836
+- Rewards/rejected: -0.7661
 - Rewards/accuracies: 0.75
-- Rewards/margins: 0.4681
-- Logps/rejected: -342.4480
-- Logps/chosen: -296.1005
-- Logits/rejected: -2.7716
-- Logits/chosen: -2.7940
+- Rewards/margins: 0.4825
+- Logps/rejected: -342.4585
+- Logps/chosen: -295.9262
+- Logits/rejected: -2.6851
+- Logits/chosen: -2.7117
 
 ## Model description
 
@@ -61,7 +61,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5794        | 0.4982 | 238  | 0.5754          | -0.2750        | -0.7430          | 0.75               | 0.4681          | -342.4480      | -296.1005    | -2.7716         | -2.7940       |
+| 0.5798        | 0.4982 | 238  | 0.5765          | -0.2836        | -0.7661          | 0.75               | 0.4825          | -342.4585      | -295.9262    | -2.6851         | -2.7117       |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.4981684981684982,
     "total_flos": 0.0,
-    "train_loss": 0.6078168283991453,
-    "train_runtime": 7171.6132,
+    "train_loss": 0.6047679316095945,
+    "train_runtime": 7201.8834,
     "train_samples": 61134,
-    "train_samples_per_second": 4.248,
+    "train_samples_per_second": 4.23,
     "train_steps_per_second": 0.033
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "data/sft/zephyr-7b-sft-1e-every25/checkpoint-800",
+  "_name_or_path": "data/sft/zephyr-7b-sft-1e-every25/checkpoint-900",
   "architectures": [
     "MistralForCausalLM"
   ],

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7ab4c3dd2c0d3fd4f7fcfff31c89a77d2f74b0fc576d9fe7599f27bd8b2a4b4b
+oid sha256:8d93eb6300b877466ea2f4e521cf819b0ef638a9ea09ad57832ad62ea8917b3f
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:222400a563ac3e0a92272027354d4791e76e7c0d11fa4bb7048704eab69b9332
+oid sha256:09174fa607bcca81805e06045dda478fdf2a5dc93bfb0a52c322bf4fa96de23a
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6966cc8a2dce33eff6646f5951b4ceaa5645114baa7ef5127ff52edcfc21f1c0
+oid sha256:81fa5d1748aec6d9e1442511af340a774590249d11e7d8f0b2e7d2f38c73ea44
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.4981684981684982,
     "total_flos": 0.0,
-    "train_loss": 0.6078168283991453,
-    "train_runtime": 7171.6132,
+    "train_loss": 0.6047679316095945,
+    "train_runtime": 7201.8834,
     "train_samples": 61134,
-    "train_samples_per_second": 4.248,
+    "train_samples_per_second": 4.23,
     "train_steps_per_second": 0.033
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0020931449502878076,
-      "grad_norm": 12.806140486914385,
+      "grad_norm": 12.610560260880211,
       "learning_rate": 2.083333333333333e-08,
-      "logits/chosen": -2.975465774536133,
-      "logits/rejected": -2.9397153854370117,
-      "logps/chosen": -317.1812744140625,
-      "logps/rejected": -362.2382507324219,
+      "logits/chosen": -2.99812388420105,
+      "logits/rejected": -2.9638350009918213,
+      "logps/chosen": -315.4977722167969,
+      "logps/rejected": -361.9093322753906,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,372 +25,372 @@
     },
     {
       "epoch": 0.020931449502878074,
-      "grad_norm": 10.350232636069011,
+      "grad_norm": 9.936649993419687,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.766873836517334,
-      "logits/rejected": -2.7434821128845215,
-      "logps/chosen": -315.6714782714844,
-      "logps/rejected": -285.2418518066406,
+      "logits/chosen": -2.788989543914795,
+      "logits/rejected": -2.7662315368652344,
+      "logps/chosen": -314.0841064453125,
+      "logps/rejected": -282.2810974121094,
       "loss": 0.693,
-      "rewards/accuracies": 0.4652777910232544,
-      "rewards/chosen": 0.0004917462356388569,
-      "rewards/margins": 0.00031292904168367386,
-      "rewards/rejected": 0.00017881726671475917,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": 0.0008651986136101186,
+      "rewards/margins": 0.0003082120092585683,
+      "rewards/rejected": 0.0005569865461438894,
       "step": 10
     },
     {
       "epoch": 0.04186289900575615,
-      "grad_norm": 12.821109233939524,
+      "grad_norm": 8.667356504170359,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.844773530960083,
-      "logits/rejected": -2.794069766998291,
-      "logps/chosen": -312.2287902832031,
-      "logps/rejected": -285.95184326171875,
-      "loss": 0.6913,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": 0.012318294495344162,
-      "rewards/margins": 0.005521962884813547,
-      "rewards/rejected": 0.0067963311448693275,
+      "logits/chosen": -2.8659820556640625,
+      "logits/rejected": -2.8171162605285645,
+      "logps/chosen": -311.6136779785156,
+      "logps/rejected": -284.09893798828125,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.013467146083712578,
+      "rewards/margins": 0.004726298153400421,
+      "rewards/rejected": 0.008740848861634731,
       "step": 20
     },
     {
       "epoch": 0.06279434850863422,
-      "grad_norm": 7.795531413975491,
+      "grad_norm": 8.034528876736365,
       "learning_rate": 4.99030821197584e-07,
-      "logits/chosen": -2.828461170196533,
-      "logits/rejected": -2.771340847015381,
-      "logps/chosen": -291.7936706542969,
-      "logps/rejected": -249.0801544189453,
-      "loss": 0.6845,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.05275644734501839,
-      "rewards/margins": 0.02750842645764351,
-      "rewards/rejected": 0.02524801716208458,
+      "logits/chosen": -2.8421683311462402,
+      "logits/rejected": -2.7854294776916504,
+      "logps/chosen": -289.38385009765625,
+      "logps/rejected": -247.28732299804688,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.05833645910024643,
+      "rewards/margins": 0.024186396971344948,
+      "rewards/rejected": 0.03415006399154663,
       "step": 30
     },
     {
       "epoch": 0.0837257980115123,
-      "grad_norm": 7.729964276089227,
+      "grad_norm": 7.833193286645188,
       "learning_rate": 4.931352528237397e-07,
-      "logits/chosen": -2.7441623210906982,
-      "logits/rejected": -2.7178843021392822,
-      "logps/chosen": -270.9248962402344,
-      "logps/rejected": -264.02130126953125,
-      "loss": 0.6709,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.047667648643255234,
-      "rewards/margins": 0.04670105502009392,
-      "rewards/rejected": 0.0009665900724940002,
+      "logits/chosen": -2.7391459941864014,
+      "logits/rejected": -2.716399669647217,
+      "logps/chosen": -269.075927734375,
+      "logps/rejected": -262.9975891113281,
+      "loss": 0.6682,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.052591562271118164,
+      "rewards/margins": 0.0480208583176136,
+      "rewards/rejected": 0.0045707011595368385,
       "step": 40
     },
     {
       "epoch": 0.10465724751439037,
-      "grad_norm": 11.62937518271555,
+      "grad_norm": 8.72813708365478,
       "learning_rate": 4.820092227512735e-07,
-      "logits/chosen": -2.778378963470459,
-      "logits/rejected": -2.7200958728790283,
-      "logps/chosen": -285.19012451171875,
-      "logps/rejected": -288.2344665527344,
-      "loss": 0.6545,
+      "logits/chosen": -2.7570505142211914,
+      "logits/rejected": -2.696572780609131,
+      "logps/chosen": -282.76092529296875,
+      "logps/rejected": -286.132568359375,
+      "loss": 0.6548,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.017544183880090714,
-      "rewards/margins": 0.08298598229885101,
-      "rewards/rejected": -0.10053016990423203,
+      "rewards/chosen": -0.010979737155139446,
+      "rewards/margins": 0.09300607442855835,
+      "rewards/rejected": -0.10398580878973007,
       "step": 50
     },
     {
       "epoch": 0.12558869701726844,
-      "grad_norm": 13.56070671513526,
+      "grad_norm": 13.480613892862241,
       "learning_rate": 4.658920803689553e-07,
-      "logits/chosen": -2.8251986503601074,
-      "logits/rejected": -2.7807350158691406,
-      "logps/chosen": -258.30609130859375,
-      "logps/rejected": -263.3247985839844,
-      "loss": 0.639,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.0033275161404162645,
-      "rewards/margins": 0.15021316707134247,
-      "rewards/rejected": -0.15354067087173462,
+      "logits/chosen": -2.789520502090454,
+      "logits/rejected": -2.7424087524414062,
+      "logps/chosen": -258.74908447265625,
+      "logps/rejected": -264.25225830078125,
+      "loss": 0.6357,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.012783573940396309,
+      "rewards/margins": 0.15775156021118164,
+      "rewards/rejected": -0.1705351322889328,
       "step": 60
     },
     {
       "epoch": 0.14652014652014653,
-      "grad_norm": 10.218106790158542,
+      "grad_norm": 10.234102018070638,
       "learning_rate": 4.4513054666826144e-07,
-      "logits/chosen": -2.790792942047119,
-      "logits/rejected": -2.7651076316833496,
-      "logps/chosen": -312.69342041015625,
-      "logps/rejected": -314.4419860839844,
-      "loss": 0.6185,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.11530323326587677,
-      "rewards/margins": 0.186301589012146,
-      "rewards/rejected": -0.3016048073768616,
+      "logits/chosen": -2.7376105785369873,
+      "logits/rejected": -2.709526300430298,
+      "logps/chosen": -311.8622131347656,
+      "logps/rejected": -316.22576904296875,
+      "loss": 0.6163,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.11481525003910065,
+      "rewards/margins": 0.20760241150856018,
+      "rewards/rejected": -0.32241764664649963,
       "step": 70
     },
     {
       "epoch": 0.1674515960230246,
-      "grad_norm": 17.77112056066791,
+      "grad_norm": 20.552548250652773,
       "learning_rate": 4.201712553872657e-07,
-      "logits/chosen": -2.862198829650879,
-      "logits/rejected": -2.790428400039673,
-      "logps/chosen": -347.1138610839844,
-      "logps/rejected": -294.9046325683594,
-      "loss": 0.6071,
+      "logits/chosen": -2.816467761993408,
+      "logits/rejected": -2.7381579875946045,
+      "logps/chosen": -341.7183837890625,
+      "logps/rejected": -293.2793884277344,
+      "loss": 0.6055,
       "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.060042936354875565,
-      "rewards/margins": 0.2697007358074188,
-      "rewards/rejected": -0.3297436833381653,
+      "rewards/chosen": -0.03545045107603073,
+      "rewards/margins": 0.28459519147872925,
+      "rewards/rejected": -0.3200456500053406,
       "step": 80
     },
     {
       "epoch": 0.18838304552590268,
-      "grad_norm": 13.034563565082575,
+      "grad_norm": 13.720188030623374,
       "learning_rate": 3.9155114477557926e-07,
-      "logits/chosen": -2.8054041862487793,
-      "logits/rejected": -2.764477252960205,
-      "logps/chosen": -276.40655517578125,
-      "logps/rejected": -307.20379638671875,
-      "loss": 0.5917,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.19227294623851776,
-      "rewards/margins": 0.30155879259109497,
-      "rewards/rejected": -0.49383172392845154,
+      "logits/chosen": -2.7560811042785645,
+      "logits/rejected": -2.7106573581695557,
+      "logps/chosen": -275.1942443847656,
+      "logps/rejected": -306.68206787109375,
+      "loss": 0.5887,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1888897567987442,
+      "rewards/margins": 0.30051952600479126,
+      "rewards/rejected": -0.48940929770469666,
       "step": 90
     },
     {
       "epoch": 0.20931449502878074,
-      "grad_norm": 16.1083959917105,
+      "grad_norm": 16.303248781308948,
       "learning_rate": 3.598859066780754e-07,
-      "logits/chosen": -2.7885613441467285,
-      "logits/rejected": -2.766613483428955,
-      "logps/chosen": -324.69219970703125,
-      "logps/rejected": -338.00518798828125,
-      "loss": 0.5974,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.1883513629436493,
-      "rewards/margins": 0.3904525637626648,
-      "rewards/rejected": -0.5788038969039917,
+      "logits/chosen": -2.7324023246765137,
+      "logits/rejected": -2.7062618732452393,
+      "logps/chosen": -322.99603271484375,
+      "logps/rejected": -339.0654602050781,
+      "loss": 0.5902,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.19065451622009277,
+      "rewards/margins": 0.39800626039505005,
+      "rewards/rejected": -0.588660717010498,
       "step": 100
     },
     {
       "epoch": 0.2302459445316588,
-      "grad_norm": 22.332005589786586,
+      "grad_norm": 20.16983103502839,
       "learning_rate": 3.2585674142717477e-07,
-      "logits/chosen": -2.8293983936309814,
-      "logits/rejected": -2.7860958576202393,
-      "logps/chosen": -325.58624267578125,
-      "logps/rejected": -338.5185546875,
-      "loss": 0.6035,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.29208147525787354,
-      "rewards/margins": 0.30902299284935,
-      "rewards/rejected": -0.6011044383049011,
+      "logits/chosen": -2.7619636058807373,
+      "logits/rejected": -2.713339328765869,
+      "logps/chosen": -338.515869140625,
+      "logps/rejected": -351.4220886230469,
+      "loss": 0.5967,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4326956868171692,
+      "rewards/margins": 0.302705854177475,
+      "rewards/rejected": -0.7354015111923218,
       "step": 110
     },
     {
       "epoch": 0.25117739403453687,
-      "grad_norm": 13.280894933092178,
+      "grad_norm": 18.288819495100896,
       "learning_rate": 2.9019570347986706e-07,
-      "logits/chosen": -2.745093822479248,
-      "logits/rejected": -2.7524163722991943,
-      "logps/chosen": -314.64752197265625,
-      "logps/rejected": -340.332275390625,
-      "loss": 0.5889,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.3470572233200073,
-      "rewards/margins": 0.3631521761417389,
-      "rewards/rejected": -0.7102094292640686,
+      "logits/chosen": -2.670757293701172,
+      "logits/rejected": -2.6768569946289062,
+      "logps/chosen": -309.2828674316406,
+      "logps/rejected": -339.7432861328125,
+      "loss": 0.579,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.33222970366477966,
+      "rewards/margins": 0.40109533071517944,
+      "rewards/rejected": -0.7333250045776367,
       "step": 120
     },
     {
       "epoch": 0.272108843537415,
-      "grad_norm": 13.497407162201402,
+      "grad_norm": 15.354064982647847,
       "learning_rate": 2.536699530523291e-07,
-      "logits/chosen": -2.8007476329803467,
-      "logits/rejected": -2.786205768585205,
-      "logps/chosen": -319.64202880859375,
-      "logps/rejected": -330.56878662109375,
-      "loss": 0.5715,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.23361213505268097,
-      "rewards/margins": 0.30114299058914185,
-      "rewards/rejected": -0.5347551107406616,
+      "logits/chosen": -2.722832679748535,
+      "logits/rejected": -2.7066872119903564,
+      "logps/chosen": -318.85833740234375,
+      "logps/rejected": -335.77911376953125,
+      "loss": 0.5666,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.2824149429798126,
+      "rewards/margins": 0.33272355794906616,
+      "rewards/rejected": -0.6151384711265564,
       "step": 130
     },
     {
       "epoch": 0.29304029304029305,
-      "grad_norm": 20.933507562985636,
+      "grad_norm": 17.28336393009731,
       "learning_rate": 2.1706525253979534e-07,
-      "logits/chosen": -2.835508108139038,
-      "logits/rejected": -2.8120312690734863,
-      "logps/chosen": -345.1457214355469,
-      "logps/rejected": -339.02618408203125,
-      "loss": 0.5833,
+      "logits/chosen": -2.7583320140838623,
+      "logits/rejected": -2.729793071746826,
+      "logps/chosen": -346.2691955566406,
+      "logps/rejected": -344.5732116699219,
+      "loss": 0.5763,
       "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.22297212481498718,
-      "rewards/margins": 0.44189220666885376,
-      "rewards/rejected": -0.6648643612861633,
+      "rewards/chosen": -0.24767926335334778,
+      "rewards/margins": 0.4790892004966736,
+      "rewards/rejected": -0.7267683744430542,
       "step": 140
     },
     {
       "epoch": 0.3139717425431711,
-      "grad_norm": 19.228587426808158,
+      "grad_norm": 19.361596281568826,
       "learning_rate": 1.8116906275593507e-07,
-      "logits/chosen": -2.821563720703125,
-      "logits/rejected": -2.793996572494507,
-      "logps/chosen": -326.88836669921875,
-      "logps/rejected": -324.4208984375,
-      "loss": 0.5803,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.378085196018219,
-      "rewards/margins": 0.3385787606239319,
-      "rewards/rejected": -0.7166639566421509,
+      "logits/chosen": -2.7468533515930176,
+      "logits/rejected": -2.7160446643829346,
+      "logps/chosen": -323.2176208496094,
+      "logps/rejected": -324.5989990234375,
+      "loss": 0.5796,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3653566539287567,
+      "rewards/margins": 0.3610517382621765,
+      "rewards/rejected": -0.7264083623886108,
       "step": 150
     },
     {
       "epoch": 0.3349031920460492,
-      "grad_norm": 19.174465914045467,
+      "grad_norm": 16.313995999292402,
       "learning_rate": 1.4675360263490295e-07,
-      "logits/chosen": -2.8593170642852783,
-      "logits/rejected": -2.820272922515869,
-      "logps/chosen": -310.5607604980469,
-      "logps/rejected": -326.0252990722656,
-      "loss": 0.582,
+      "logits/chosen": -2.7838118076324463,
+      "logits/rejected": -2.7365283966064453,
+      "logps/chosen": -319.6917724609375,
+      "logps/rejected": -340.581787109375,
+      "loss": 0.5759,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.3000241219997406,
-      "rewards/margins": 0.4080016016960144,
-      "rewards/rejected": -0.7080257534980774,
+      "rewards/chosen": -0.40762004256248474,
+      "rewards/margins": 0.4462064802646637,
+      "rewards/rejected": -0.8538265228271484,
       "step": 160
     },
     {
       "epoch": 0.35583464154892724,
-      "grad_norm": 18.167727970452024,
+      "grad_norm": 22.925451744364146,
       "learning_rate": 1.1455923682523475e-07,
-      "logits/chosen": -2.747973918914795,
-      "logits/rejected": -2.7372567653656006,
-      "logps/chosen": -295.9173278808594,
-      "logps/rejected": -303.07861328125,
-      "loss": 0.587,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.3046445846557617,
-      "rewards/margins": 0.3219013214111328,
-      "rewards/rejected": -0.6265459060668945,
+      "logits/chosen": -2.659465789794922,
+      "logits/rejected": -2.644275188446045,
+      "logps/chosen": -311.614990234375,
+      "logps/rejected": -323.3099670410156,
+      "loss": 0.585,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.4685409665107727,
+      "rewards/margins": 0.36185160279273987,
+      "rewards/rejected": -0.8303925395011902,
       "step": 170
     },
     {
       "epoch": 0.37676609105180536,
-      "grad_norm": 14.998685600639057,
+      "grad_norm": 13.975706307085083,
       "learning_rate": 8.527854855097224e-08,
-      "logits/chosen": -2.834468364715576,
-      "logits/rejected": -2.8099303245544434,
-      "logps/chosen": -307.76336669921875,
-      "logps/rejected": -314.2436828613281,
-      "loss": 0.5608,
+      "logits/chosen": -2.7547802925109863,
+      "logits/rejected": -2.7258901596069336,
+      "logps/chosen": -313.0289001464844,
+      "logps/rejected": -321.9787902832031,
+      "loss": 0.5538,
       "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.32118621468544006,
-      "rewards/margins": 0.3334035873413086,
-      "rewards/rejected": -0.6545897722244263,
+      "rewards/chosen": -0.37977224588394165,
+      "rewards/margins": 0.3600301146507263,
+      "rewards/rejected": -0.739802360534668,
       "step": 180
     },
     {
       "epoch": 0.3976975405546834,
-      "grad_norm": 15.220786257352342,
+      "grad_norm": 16.933160186335314,
       "learning_rate": 5.9541440373546445e-08,
-      "logits/chosen": -2.776503801345825,
-      "logits/rejected": -2.7547481060028076,
-      "logps/chosen": -294.9460754394531,
-      "logps/rejected": -318.6103820800781,
-      "loss": 0.5767,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.32699090242385864,
-      "rewards/margins": 0.2969752252101898,
-      "rewards/rejected": -0.6239660978317261,
+      "logits/chosen": -2.689134120941162,
+      "logits/rejected": -2.664074420928955,
+      "logps/chosen": -296.32672119140625,
+      "logps/rejected": -319.95001220703125,
+      "loss": 0.5755,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.34621429443359375,
+      "rewards/margins": 0.2947639226913452,
+      "rewards/rejected": -0.6409782767295837,
       "step": 190
     },
     {
       "epoch": 0.4186289900575615,
-      "grad_norm": 19.32668573129983,
+      "grad_norm": 21.908273233954745,
       "learning_rate": 3.790158337517127e-08,
-      "logits/chosen": -2.746075391769409,
-      "logits/rejected": -2.7452704906463623,
-      "logps/chosen": -289.711669921875,
-      "logps/rejected": -340.4427795410156,
-      "loss": 0.5829,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.31456780433654785,
-      "rewards/margins": 0.3399312496185303,
-      "rewards/rejected": -0.6544990539550781,
+      "logits/chosen": -2.653926372528076,
+      "logits/rejected": -2.6524300575256348,
+      "logps/chosen": -289.92926025390625,
+      "logps/rejected": -341.435546875,
+      "loss": 0.5823,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.3269258737564087,
+      "rewards/margins": 0.3465423882007599,
+      "rewards/rejected": -0.6734683513641357,
       "step": 200
     },
     {
       "epoch": 0.43956043956043955,
-      "grad_norm": 15.573661203693316,
+      "grad_norm": 18.994509414379117,
       "learning_rate": 2.0824506276503894e-08,
-      "logits/chosen": -2.794292688369751,
-      "logits/rejected": -2.7379298210144043,
-      "logps/chosen": -354.16455078125,
-      "logps/rejected": -326.73974609375,
-      "loss": 0.5882,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.3771507441997528,
-      "rewards/margins": 0.36129769682884216,
-      "rewards/rejected": -0.7384485006332397,
+      "logits/chosen": -2.7132248878479004,
+      "logits/rejected": -2.648906946182251,
+      "logps/chosen": -352.23211669921875,
+      "logps/rejected": -327.8628234863281,
+      "loss": 0.5851,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.38004761934280396,
+      "rewards/margins": 0.3670490086078644,
+      "rewards/rejected": -0.7470966577529907,
       "step": 210
     },
     {
       "epoch": 0.4604918890633176,
-      "grad_norm": 15.206763088206086,
+      "grad_norm": 16.241884556359377,
       "learning_rate": 8.677580722139671e-09,
-      "logits/chosen": -2.721893310546875,
-      "logits/rejected": -2.735137939453125,
-      "logps/chosen": -293.5481872558594,
-      "logps/rejected": -323.8240661621094,
-      "loss": 0.5737,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.3610805869102478,
-      "rewards/margins": 0.3861430883407593,
-      "rewards/rejected": -0.7472237348556519,
+      "logits/chosen": -2.635542154312134,
+      "logits/rejected": -2.6469016075134277,
+      "logps/chosen": -288.9513244628906,
+      "logps/rejected": -323.00494384765625,
+      "loss": 0.573,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.3638390898704529,
+      "rewards/margins": 0.3917424976825714,
+      "rewards/rejected": -0.7555815577507019,
       "step": 220
     },
     {
       "epoch": 0.48142333856619574,
-      "grad_norm": 16.74159983073626,
+      "grad_norm": 22.899672406649472,
       "learning_rate": 1.722118176089915e-09,
-      "logits/chosen": -2.821955919265747,
-      "logits/rejected": -2.7882137298583984,
-      "logps/chosen": -294.47637939453125,
-      "logps/rejected": -329.96240234375,
-      "loss": 0.5794,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.27040696144104004,
-      "rewards/margins": 0.5722111463546753,
-      "rewards/rejected": -0.8426181077957153,
+      "logits/chosen": -2.742950439453125,
+      "logits/rejected": -2.702911376953125,
+      "logps/chosen": -292.0196533203125,
+      "logps/rejected": -330.7127380371094,
+      "loss": 0.5798,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.25979113578796387,
+      "rewards/margins": 0.5986371040344238,
+      "rewards/rejected": -0.8584282994270325,
       "step": 230
     },
     {
       "epoch": 0.4981684981684982,
-      "eval_logits/chosen": -2.794038772583008,
-      "eval_logits/rejected": -2.771556854248047,
-      "eval_logps/chosen": -296.10052490234375,
-      "eval_logps/rejected": -342.4480285644531,
-      "eval_loss": 0.5754034519195557,
+      "eval_logits/chosen": -2.711683750152588,
+      "eval_logits/rejected": -2.68514084815979,
+      "eval_logps/chosen": -295.9261779785156,
+      "eval_logps/rejected": -342.4585266113281,
+      "eval_loss": 0.576471209526062,
       "eval_rewards/accuracies": 0.75,
-      "eval_rewards/chosen": -0.27497145533561707,
-      "eval_rewards/margins": 0.4680546224117279,
-      "eval_rewards/rejected": -0.743026077747345,
-      "eval_runtime": 169.1974,
-      "eval_samples_per_second": 11.821,
-      "eval_steps_per_second": 0.189,
+      "eval_rewards/chosen": -0.283584326505661,
+      "eval_rewards/margins": 0.48247623443603516,
+      "eval_rewards/rejected": -0.7660605311393738,
+      "eval_runtime": 168.4238,
+      "eval_samples_per_second": 11.875,
+      "eval_steps_per_second": 0.19,
       "step": 238
     },
     {
       "epoch": 0.4981684981684982,
       "step": 238,
       "total_flos": 0.0,
-      "train_loss": 0.6078168283991453,
-      "train_runtime": 7171.6132,
-      "train_samples_per_second": 4.248,
+      "train_loss": 0.6047679316095945,
+      "train_runtime": 7201.8834,
+      "train_samples_per_second": 4.23,
       "train_steps_per_second": 0.033
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:99dcad4a7ce8ebd62f827204bb8667baa90a4ae760788ca43c162ee9caab450d
+oid sha256:13885dc2cfddf0eb7080d5bad9f264e574a65e3f8468c00bc6a65382edb01768
 size 7672