Hugging Face's logo

allen0909
/
ROE_Patent_Breeze7B_20250124_Type2

PEFT
TensorBoard
Safetensors
trl
sft
Generated from Trainer
Model card Files
xet
Metrics Community
allen0909 commited on
Commit
7030e3c
·
verified ·
1 Parent(s): 91e5017

Training in progress, step 100

Browse files
Files changed (23) hide show
  1. README.md +80 -0
  2. adapter_config.json +33 -0
  3. adapter_model.safetensors +3 -0
  4. added_tokens.json +4 -0
  5. runs/Jan24_01-15-39_09bedd6e5be9/events.out.tfevents.1737681346.09bedd6e5be9.1285.0 +3 -0
  6. runs/Jan24_01-19-46_09bedd6e5be9/events.out.tfevents.1737681590.09bedd6e5be9.5589.0 +3 -0
  7. runs/Jan24_01-33-20_09bedd6e5be9/events.out.tfevents.1737682404.09bedd6e5be9.9636.0 +3 -0
  8. runs/Jan24_01-48-45_09bedd6e5be9/events.out.tfevents.1737683329.09bedd6e5be9.13608.0 +3 -0
  9. runs/Jan24_02-13-45_09bedd6e5be9/events.out.tfevents.1737684837.09bedd6e5be9.20134.0 +3 -0
  10. runs/Jan24_02-39-06_09bedd6e5be9/events.out.tfevents.1737686361.09bedd6e5be9.26763.0 +3 -0
  11. runs/Jan24_02-40-36_09bedd6e5be9/events.out.tfevents.1737686440.09bedd6e5be9.27537.0 +3 -0
  12. runs/Jan24_02-42-57_09bedd6e5be9/events.out.tfevents.1737686581.09bedd6e5be9.28547.0 +3 -0
  13. runs/Jan24_02-45-27_09bedd6e5be9/events.out.tfevents.1737686734.09bedd6e5be9.29478.0 +3 -0
  14. runs/Jan24_03-13-46_09bedd6e5be9/events.out.tfevents.1737688430.09bedd6e5be9.36957.0 +3 -0
  15. runs/Jan24_03-15-24_09bedd6e5be9/events.out.tfevents.1737688529.09bedd6e5be9.37694.0 +3 -0
  16. runs/Jan24_03-19-27_09bedd6e5be9/events.out.tfevents.1737688771.09bedd6e5be9.39124.0 +3 -0
  17. runs/Jan24_03-38-10_09bedd6e5be9/events.out.tfevents.1737689894.09bedd6e5be9.44108.0 +3 -0
  18. runs/Jan24_03-59-20_09bedd6e5be9/events.out.tfevents.1737691165.09bedd6e5be9.49683.0 +3 -0
  19. special_tokens_map.json +24 -0
  20. tokenizer.json +0 -0
  21. tokenizer.model +3 -0
  22. tokenizer_config.json +62 -0
  23. training_args.bin +3 -0
README.md ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ license: apache-2.0
4
+ base_model: MediaTek-Research/Breeze-7B-Instruct-v1_0
5
+ tags:
6
+ - trl
7
+ - sft
8
+ - generated_from_trainer
9
+ model-index:
10
+ - name: ROE_Patent_Breeze7B_20250124_Type1
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # ROE_Patent_Breeze7B_20250124_Type1
18
+
19
+ This model is a fine-tuned version of [MediaTek-Research/Breeze-7B-Instruct-v1_0](https://huggingface.co/MediaTek-Research/Breeze-7B-Instruct-v1_0) on the None dataset.
20
+ It achieves the following results on the evaluation set:
21
+ - Loss: 0.7582
22
+
23
+ ## Model description
24
+
25
+ More information needed
26
+
27
+ ## Intended uses & limitations
28
+
29
+ More information needed
30
+
31
+ ## Training and evaluation data
32
+
33
+ More information needed
34
+
35
+ ## Training procedure
36
+
37
+ ### Training hyperparameters
38
+
39
+ The following hyperparameters were used during training:
40
+ - learning_rate: 5e-05
41
+ - train_batch_size: 1
42
+ - eval_batch_size: 8
43
+ - seed: 42
44
+ - gradient_accumulation_steps: 8
45
+ - total_train_batch_size: 8
46
+ - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
47
+ - lr_scheduler_type: cosine
48
+ - lr_scheduler_warmup_ratio: 0.05
49
+ - num_epochs: 17
50
+ - mixed_precision_training: Native AMP
51
+
52
+ ### Training results
53
+
54
+ | Training Loss | Epoch | Step | Validation Loss |
55
+ |:-------------:|:-------:|:----:|:---------------:|
56
+ | 1.0139 | 1.0 | 13 | 1.0349 |
57
+ | 0.9149 | 2.0 | 26 | 0.9424 |
58
+ | 0.8664 | 3.0 | 39 | 0.8711 |
59
+ | 0.7648 | 4.0 | 52 | 0.8391 |
60
+ | 0.7454 | 5.0 | 65 | 0.8082 |
61
+ | 0.7115 | 6.0 | 78 | 0.7896 |
62
+ | 0.6892 | 7.0 | 91 | 0.7794 |
63
+ | 0.6379 | 8.0 | 104 | 0.7733 |
64
+ | 0.6592 | 9.0 | 117 | 0.7650 |
65
+ | 0.6346 | 10.0 | 130 | 0.7655 |
66
+ | 0.6214 | 11.0 | 143 | 0.7607 |
67
+ | 0.6093 | 12.0 | 156 | 0.7601 |
68
+ | 0.6085 | 13.0 | 169 | 0.7590 |
69
+ | 0.6268 | 14.0 | 182 | 0.7585 |
70
+ | 0.609 | 15.0 | 195 | 0.7575 |
71
+ | 0.6206 | 15.7273 | 204 | 0.7582 |
72
+
73
+
74
+ ### Framework versions
75
+
76
+ - PEFT 0.14.1.dev0
77
+ - Transformers 4.48.1
78
+ - Pytorch 2.5.1+cu121
79
+ - Datasets 3.2.0
80
+ - Tokenizers 0.21.0
adapter_config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "MediaTek-Research/Breeze-7B-Instruct-v1_0",
5
+ "bias": "none",
6
+ "corda_config": null,
7
+ "eva_config": null,
8
+ "exclude_modules": null,
9
+ "fan_in_fan_out": false,
10
+ "inference_mode": true,
11
+ "init_lora_weights": true,
12
+ "layer_replication": null,
13
+ "layers_pattern": null,
14
+ "layers_to_transform": null,
15
+ "loftq_config": {},
16
+ "lora_alpha": 16,
17
+ "lora_bias": false,
18
+ "lora_dropout": 0.1,
19
+ "megatron_config": null,
20
+ "megatron_core": "megatron.core",
21
+ "modules_to_save": null,
22
+ "peft_type": "LORA",
23
+ "r": 8,
24
+ "rank_pattern": {},
25
+ "revision": null,
26
+ "target_modules": [
27
+ "q_proj",
28
+ "v_proj"
29
+ ],
30
+ "task_type": "CAUSAL_LM",
31
+ "use_dora": false,
32
+ "use_rslora": false
33
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:11697d2d7cb05985db9535a777cee3be90ce49dec9c73f41891fffb7a5cf5aa4
3
+ size 13648432
added_tokens.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "<EOD>": 61873,
3
+ "<PAD>": 61874
4
+ }
runs/Jan24_01-15-39_09bedd6e5be9/events.out.tfevents.1737681346.09bedd6e5be9.1285.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da9e77645be19267e8cb46ece32efabb93e347fcab3d444e376c768f0b08f1a5
3
+ size 6901
runs/Jan24_01-19-46_09bedd6e5be9/events.out.tfevents.1737681590.09bedd6e5be9.5589.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f02a53d96ba07b8b903c951af37dc34300eddc87aa2b9cbb6a5ef42a8b19891
3
+ size 10153
runs/Jan24_01-33-20_09bedd6e5be9/events.out.tfevents.1737682404.09bedd6e5be9.9636.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e99e8303b5d5de21d87246364e6d956e5a99567197e1b574d8f73d1cd37531b3
3
+ size 11799
runs/Jan24_01-48-45_09bedd6e5be9/events.out.tfevents.1737683329.09bedd6e5be9.13608.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:697cdb5ea31c03f4e64c09585eededd4586ae50d5b453d0d9d4f48dd2d864853
3
+ size 15256
runs/Jan24_02-13-45_09bedd6e5be9/events.out.tfevents.1737684837.09bedd6e5be9.20134.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0772566eda7bc48fbc1492c0e23db6a2868038ed8e37169928bd6bff20b1fe0
3
+ size 15254
runs/Jan24_02-39-06_09bedd6e5be9/events.out.tfevents.1737686361.09bedd6e5be9.26763.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e3e396fac7ae65ac1de6ad6f79270d4917de5d64be056e2d58e5f251e791c98
3
+ size 4184
runs/Jan24_02-40-36_09bedd6e5be9/events.out.tfevents.1737686440.09bedd6e5be9.27537.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b606a612f51214cb5ef2e9b5a62d1fd605293e4501f80d21a4e3b75d0a9f24d5
3
+ size 6156
runs/Jan24_02-42-57_09bedd6e5be9/events.out.tfevents.1737686581.09bedd6e5be9.28547.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f354804fe06fbb8c7bf1baa769309b4c3b9fd4d0ba0d6476d57b1fbeb00defc7
3
+ size 6422
runs/Jan24_02-45-27_09bedd6e5be9/events.out.tfevents.1737686734.09bedd6e5be9.29478.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b55ebb2323398c6cf2abedb8dfb29124f75e3ed4ace0d7697c5df80dd0f8991a
3
+ size 16423
runs/Jan24_03-13-46_09bedd6e5be9/events.out.tfevents.1737688430.09bedd6e5be9.36957.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:073bc371263756c8cf501d85a5b9ed10251616b55ffd5d7673d6973538bb80ca
3
+ size 4184
runs/Jan24_03-15-24_09bedd6e5be9/events.out.tfevents.1737688529.09bedd6e5be9.37694.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:55825d003c59720271ca84e7d5ca15ff2c725a537e2dea16d5eade7874db7e99
3
+ size 5957
runs/Jan24_03-19-27_09bedd6e5be9/events.out.tfevents.1737688771.09bedd6e5be9.39124.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb4f70bc47410f4ea49cbadda68f60ada720d01d270a24f3746482bfdc28a829
3
+ size 13802
runs/Jan24_03-38-10_09bedd6e5be9/events.out.tfevents.1737689894.09bedd6e5be9.44108.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a7d0e11a51f51859e7004403e5f8596743d74afbd912755e7c606aab11bb9646
3
+ size 14766
runs/Jan24_03-59-20_09bedd6e5be9/events.out.tfevents.1737691165.09bedd6e5be9.49683.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49d9d4af3bd71d74939429a47152b5e19649e6c4ddac155172a2db5c0c677c94
3
+ size 9881
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9298e56c094f0d30431b0e52ad53287f0cadc99ac8ca17cc2144b0eb4753f130
3
+ size 911034
tokenizer_config.json ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": true,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ },
30
+ "61873": {
31
+ "content": "<EOD>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false,
36
+ "special": true
37
+ },
38
+ "61874": {
39
+ "content": "<PAD>",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": false,
43
+ "single_word": false,
44
+ "special": true
45
+ }
46
+ },
47
+ "bos_token": "<s>",
48
+ "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'].strip() %}{% else %}{% set loop_messages = messages %}{% set system_message = 'You are a helpful AI assistant built by MediaTek Research. The user you are helping speaks Traditional Chinese and comes from Taiwan.' %}{% endif %}{{ bos_token }} {{ system_message }} {% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/... or system/user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ ' [INST] ' + message['content'] + ' [/INST] ' }}{% elif message['role'] == 'assistant' %}{{ message['content'] }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
49
+ "clean_up_tokenization_spaces": false,
50
+ "eos_token": "</s>",
51
+ "extra_special_tokens": {},
52
+ "legacy": true,
53
+ "model_max_length": 1000000000000000019884624838656,
54
+ "pad_token": "</s>",
55
+ "padding": true,
56
+ "sp_model_kwargs": {},
57
+ "spaces_between_special_tokens": false,
58
+ "tokenizer_class": "LlamaTokenizer",
59
+ "unk_token": "<unk>",
60
+ "use_default_system_prompt": false,
61
+ "use_fast": true
62
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:31f309398321232a96d2aff718d0d5517cb0a8915791cbf914f669cb18e13cc6
3
+ size 5624