Model save

README.md

@@ -15,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.3740
+- Loss: 0.9900
 
 ## Model description
 
@@ -39,10 +39,8 @@ The following hyperparameters were used during training:
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 8
 - gradient_accumulation_steps: 128
-- total_train_batch_size: 4096
-- total_eval_batch_size: 64
+- total_train_batch_size: 512
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - num_epochs: 1
@@ -51,7 +49,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 1.3896        | 0.67  | 34   | 1.3738          |
+| 0.9866        | 0.67  | 272  | 0.9900          |
 
 
 ### Framework versions

adapter_config.json

@@ -16,10 +16,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "k_proj",
     "q_proj",
-    "o_proj",
     "v_proj",
-    "k_proj"
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7284f5c4be8442b483b3fa90a1f55f3361746d0ace5d6c819b5fabcc5f07e49c
-size 109086672
+oid sha256:8a5f251458b29980ed03af6c418d5dac972c35d885863f05cfb1a03e5c568a28
+size 218138576

all_results.json

@@ -1,13 +1,13 @@
 {
     "epoch": 0.67,
-    "eval_loss": 1.3739680051803589,
-    "eval_runtime": 805.5844,
+    "eval_loss": 0.9899652600288391,
+    "eval_runtime": 5691.7746,
     "eval_samples": 23110,
-    "eval_samples_per_second": 28.687,
-    "eval_steps_per_second": 0.449,
-    "train_loss": 1.4923833398257984,
-    "train_runtime": 25806.7327,
+    "eval_samples_per_second": 4.06,
+    "eval_steps_per_second": 0.508,
+    "train_loss": 1.009579424472416,
+    "train_runtime": 170599.5352,
     "train_samples": 207865,
-    "train_samples_per_second": 8.055,
+    "train_samples_per_second": 1.218,
     "train_steps_per_second": 0.002
 }

config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.35.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

eval_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.67,
-    "eval_loss": 1.3739680051803589,
-    "eval_runtime": 805.5844,
+    "eval_loss": 0.9899652600288391,
+    "eval_runtime": 5691.7746,
     "eval_samples": 23110,
-    "eval_samples_per_second": 28.687,
-    "eval_steps_per_second": 0.449
+    "eval_samples_per_second": 4.06,
+    "eval_steps_per_second": 0.508
 }

runs/Jan08_20-34-16_zf-eiger-finetune-debug/events.out.tfevents.1704746103.zf-eiger-finetune-debug.26752.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a124c0ce63725083e991e89f4f2c671359eb5612730ccf84009123c681bae2fa
+size 13453

runs/Jan08_20-34-16_zf-eiger-finetune-debug/events.out.tfevents.1704922394.zf-eiger-finetune-debug.26752.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8163dbd0ebb048b150fca051db4743d0fa3b272e8db28f032befc5f39f0bd139
+size 359

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.67,
-    "train_loss": 1.4923833398257984,
-    "train_runtime": 25806.7327,
+    "train_loss": 1.009579424472416,
+    "train_runtime": 170599.5352,
     "train_samples": 207865,
-    "train_samples_per_second": 8.055,
+    "train_samples_per_second": 1.218,
     "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -1,78 +1,366 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6699507389162561,
+  "epoch": 0.6699636307656782,
   "eval_steps": 500,
-  "global_step": 34,
+  "global_step": 272,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.02,
-      "learning_rate": 1.9980267284282718e-05,
-      "loss": 1.6872,
+      "epoch": 0.0,
+      "learning_rate": 1.999969914479768e-05,
+      "loss": 1.1295,
       "step": 1
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 1.9510565162951538e-05,
-      "loss": 1.6736,
+      "epoch": 0.01,
+      "learning_rate": 1.9992479525042305e-05,
+      "loss": 1.1199,
       "step": 5
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 1.8090169943749477e-05,
-      "loss": 1.5943,
+      "epoch": 0.02,
+      "learning_rate": 1.996992941167792e-05,
+      "loss": 1.1063,
       "step": 10
     },
     {
-      "epoch": 0.3,
-      "learning_rate": 1.5877852522924733e-05,
-      "loss": 1.5117,
+      "epoch": 0.04,
+      "learning_rate": 1.9932383577419432e-05,
+      "loss": 1.0914,
       "step": 15
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 1.3090169943749475e-05,
-      "loss": 1.4483,
+      "epoch": 0.05,
+      "learning_rate": 1.9879898494768093e-05,
+      "loss": 1.094,
       "step": 20
     },
     {
-      "epoch": 0.49,
-      "learning_rate": 1e-05,
-      "loss": 1.4116,
+      "epoch": 0.06,
+      "learning_rate": 1.9812553106273848e-05,
+      "loss": 1.0659,
       "step": 25
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 6.909830056250527e-06,
-      "loss": 1.3896,
+      "epoch": 0.07,
+      "learning_rate": 1.973044870579824e-05,
+      "loss": 1.0527,
       "step": 30
     },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.9633708786158803e-05,
+      "loss": 1.0503,
+      "step": 35
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.9522478853384154e-05,
+      "loss": 1.0305,
+      "step": 40
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.9396926207859085e-05,
+      "loss": 1.0268,
+      "step": 45
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9257239692688907e-05,
+      "loss": 1.0291,
+      "step": 50
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.9103629409661468e-05,
+      "loss": 1.02,
+      "step": 55
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.8936326403234125e-05,
+      "loss": 1.0127,
+      "step": 60
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.8755582313020912e-05,
+      "loss": 1.0078,
+      "step": 65
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.8561668995302668e-05,
+      "loss": 1.0003,
+      "step": 70
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.8354878114129368e-05,
+      "loss": 1.0081,
+      "step": 75
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.8135520702629677e-05,
+      "loss": 1.0157,
+      "step": 80
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.7903926695187595e-05,
+      "loss": 1.0118,
+      "step": 85
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.766044443118978e-05,
+      "loss": 1.0011,
+      "step": 90
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.740544013109005e-05,
+      "loss": 0.9919,
+      "step": 95
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.7139297345578992e-05,
+      "loss": 1.0025,
+      "step": 100
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.686241637868734e-05,
+      "loss": 1.0187,
+      "step": 105
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.657521368569064e-05,
+      "loss": 0.9959,
+      "step": 110
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.627812124672099e-05,
+      "loss": 0.9936,
+      "step": 115
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.5971585917027864e-05,
+      "loss": 0.9981,
+      "step": 120
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.5656068754865388e-05,
+      "loss": 0.9953,
+      "step": 125
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.5332044328016916e-05,
+      "loss": 1.0044,
+      "step": 130
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.5000000000000002e-05,
+      "loss": 1.0033,
+      "step": 135
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.4660435197025391e-05,
+      "loss": 0.9901,
+      "step": 140
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.4313860656812537e-05,
+      "loss": 0.9965,
+      "step": 145
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.396079766039157e-05,
+      "loss": 0.9915,
+      "step": 150
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.3601777248047105e-05,
+      "loss": 0.9861,
+      "step": 155
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.3237339420583213e-05,
+      "loss": 0.9757,
+      "step": 160
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.2868032327110904e-05,
+      "loss": 0.9944,
+      "step": 165
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.2494411440579814e-05,
+      "loss": 0.9963,
+      "step": 170
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.211703872229411e-05,
+      "loss": 0.9817,
+      "step": 175
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.1736481776669307e-05,
+      "loss": 0.9881,
+      "step": 180
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.1353312997501313e-05,
+      "loss": 0.9965,
+      "step": 185
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.0968108707031792e-05,
+      "loss": 0.9922,
+      "step": 190
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.0581448289104759e-05,
+      "loss": 0.9934,
+      "step": 195
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.0193913317718245e-05,
+      "loss": 0.9823,
+      "step": 200
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 9.806086682281759e-06,
+      "loss": 0.988,
+      "step": 205
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.418551710895243e-06,
+      "loss": 0.9867,
+      "step": 210
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.03189129296821e-06,
+      "loss": 0.9983,
+      "step": 215
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 8.646687002498692e-06,
+      "loss": 0.9866,
+      "step": 220
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.263518223330698e-06,
+      "loss": 0.9918,
+      "step": 225
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 7.882961277705897e-06,
+      "loss": 0.9901,
+      "step": 230
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 7.505588559420188e-06,
+      "loss": 0.987,
+      "step": 235
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 7.131967672889101e-06,
+      "loss": 0.9941,
+      "step": 240
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 6.762660579416791e-06,
+      "loss": 0.9968,
+      "step": 245
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.3982227519528986e-06,
+      "loss": 0.9842,
+      "step": 250
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 6.039202339608432e-06,
+      "loss": 0.9788,
+      "step": 255
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 5.686139343187468e-06,
+      "loss": 0.9929,
+      "step": 260
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 5.339564802974615e-06,
+      "loss": 0.9835,
+      "step": 265
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.000000000000003e-06,
+      "loss": 0.9866,
+      "step": 270
+    },
     {
       "epoch": 0.67,
-      "eval_loss": 1.3737928867340088,
-      "eval_runtime": 807.3041,
-      "eval_samples_per_second": 28.626,
-      "eval_steps_per_second": 0.448,
-      "step": 34
+      "eval_loss": 0.9899651408195496,
+      "eval_runtime": 5696.8421,
+      "eval_samples_per_second": 4.057,
+      "eval_steps_per_second": 0.507,
+      "step": 272
     },
     {
       "epoch": 0.67,
-      "step": 34,
-      "total_flos": 5.175876094263296e+16,
-      "train_loss": 1.4923833398257984,
-      "train_runtime": 25806.7327,
-      "train_samples_per_second": 8.055,
+      "step": 272,
+      "total_flos": 1.2275875027210994e+19,
+      "train_loss": 1.009579424472416,
+      "train_runtime": 170599.5352,
+      "train_samples_per_second": 1.218,
       "train_steps_per_second": 0.002
     }
   ],
   "logging_steps": 5,
-  "max_steps": 50,
+  "max_steps": 405,
   "num_train_epochs": 1,
   "save_steps": 500,
-  "total_flos": 5.175876094263296e+16,
+  "total_flos": 1.2275875027210994e+19,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19b7928b2959ad96d76ef30e0df163ce63a284cb8aabe667a31252f96a55303e
-size 5624
+oid sha256:cfd5d5bfedefa8f93d7df86ccb045bae5bafacb71d8b896a3b42c0d14b25448d
+size 4664