Buffett666 commited on
Commit
bf365e6
·
verified ·
1 Parent(s): 7f1c47a

Training in progress, epoch 1

Browse files
config.json ADDED
@@ -0,0 +1,221 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "ntnu-smil/phi-4-mm-lttc-binary-integrated-1764",
3
+ "architectures": [
4
+ "Phi4MMForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "audio_processor": {
9
+ "config": {
10
+ "activation": "swish",
11
+ "activation_checkpointing": {
12
+ "interval": 1,
13
+ "module": "transformer",
14
+ "offload": false
15
+ },
16
+ "attention_dim": 1024,
17
+ "attention_heads": 16,
18
+ "batch_norm": false,
19
+ "bias_in_glu": true,
20
+ "causal": true,
21
+ "chunk_size": -1,
22
+ "cnn_layer_norm": true,
23
+ "conv_activation": "swish",
24
+ "conv_glu_type": "swish",
25
+ "depthwise_multiplier": 1,
26
+ "depthwise_seperable_out_channel": 1024,
27
+ "dropout_rate": 0.0,
28
+ "encoder_embedding_config": {
29
+ "input_size": 80
30
+ },
31
+ "ext_pw_kernel_size": 1,
32
+ "ext_pw_out_channel": 1024,
33
+ "input_layer": "nemo_conv",
34
+ "input_size": 80,
35
+ "kernel_size": 3,
36
+ "left_chunk": 18,
37
+ "linear_units": 1536,
38
+ "nemo_conv_settings": {
39
+ "conv_channels": 1024
40
+ },
41
+ "num_blocks": 24,
42
+ "relative_attention_bias_args": {
43
+ "t5_bias_max_distance": 500,
44
+ "type": "t5"
45
+ },
46
+ "time_reduction": 8
47
+ },
48
+ "name": "cascades"
49
+ },
50
+ "auto_map": {
51
+ "AutoConfig": "configuration_phi4mm.Phi4MMConfig",
52
+ "AutoModelForCausalLM": "modeling_phi4mm.Phi4MMForCausalLM",
53
+ "AutoTokenizer": "microsoft/Phi-4-multimodal-instruct--Xenova/gpt-4o"
54
+ },
55
+ "bos_token_id": 199999,
56
+ "embd_layer": {
57
+ "audio_embd_layer": {
58
+ "compression_rate": 8,
59
+ "downsample_rate": 1,
60
+ "embedding_cls": "audio",
61
+ "enable_gradient_checkpointing": true,
62
+ "projection_cls": "mlp",
63
+ "use_conv_downsample": false,
64
+ "use_qformer": false
65
+ },
66
+ "embedding_cls": "image_audio",
67
+ "image_embd_layer": {
68
+ "crop_size": 448,
69
+ "embedding_cls": "tune_image",
70
+ "enable_gradient_checkpointing": true,
71
+ "hd_transform_order": "sub_glb",
72
+ "image_token_compression_cls": "avg_pool_2d",
73
+ "projection_cls": "mlp",
74
+ "use_hd_transform": true,
75
+ "with_learnable_separator": true
76
+ }
77
+ },
78
+ "embd_pdrop": 0.0,
79
+ "eos_token_id": 199999,
80
+ "full_attn_mod": 1,
81
+ "hidden_act": "silu",
82
+ "hidden_size": 3072,
83
+ "img_processor": null,
84
+ "initializer_range": 0.02,
85
+ "intermediate_size": 8192,
86
+ "interpolate_factor": 1,
87
+ "lm_head_bias": false,
88
+ "max_position_embeddings": 131072,
89
+ "mlp_bias": false,
90
+ "model_type": "phi4mm",
91
+ "num_attention_heads": 24,
92
+ "num_hidden_layers": 32,
93
+ "num_key_value_heads": 8,
94
+ "original_max_position_embeddings": 4096,
95
+ "pad_token_id": 199999,
96
+ "partial_rotary_factor": 0.75,
97
+ "resid_pdrop": 0.0,
98
+ "rms_norm_eps": 1e-05,
99
+ "rope_scaling": {
100
+ "long_factor": [
101
+ 1,
102
+ 1.118320672,
103
+ 1.250641126,
104
+ 1.398617824,
105
+ 1.564103225,
106
+ 1.74916897,
107
+ 1.956131817,
108
+ 2.187582649,
109
+ 2.446418898,
110
+ 2.735880826,
111
+ 3.059592084,
112
+ 3.421605075,
113
+ 3.826451687,
114
+ 4.279200023,
115
+ 4.785517845,
116
+ 5.351743533,
117
+ 5.984965424,
118
+ 6.693110555,
119
+ 7.485043894,
120
+ 8.370679318,
121
+ 9.36110372,
122
+ 10.4687158,
123
+ 11.70738129,
124
+ 13.09260651,
125
+ 14.64173252,
126
+ 16.37415215,
127
+ 18.31155283,
128
+ 20.47818807,
129
+ 22.90118105,
130
+ 25.61086418,
131
+ 28.64115884,
132
+ 32.03,
133
+ 32.1,
134
+ 32.13,
135
+ 32.23,
136
+ 32.6,
137
+ 32.61,
138
+ 32.64,
139
+ 32.66,
140
+ 32.7,
141
+ 32.71,
142
+ 32.93,
143
+ 32.97,
144
+ 33.28,
145
+ 33.49,
146
+ 33.5,
147
+ 44.16,
148
+ 47.77
149
+ ],
150
+ "short_factor": [
151
+ 1.0,
152
+ 1.0,
153
+ 1.0,
154
+ 1.0,
155
+ 1.0,
156
+ 1.0,
157
+ 1.0,
158
+ 1.0,
159
+ 1.0,
160
+ 1.0,
161
+ 1.0,
162
+ 1.0,
163
+ 1.0,
164
+ 1.0,
165
+ 1.0,
166
+ 1.0,
167
+ 1.0,
168
+ 1.0,
169
+ 1.0,
170
+ 1.0,
171
+ 1.0,
172
+ 1.0,
173
+ 1.0,
174
+ 1.0,
175
+ 1.0,
176
+ 1.0,
177
+ 1.0,
178
+ 1.0,
179
+ 1.0,
180
+ 1.0,
181
+ 1.0,
182
+ 1.0,
183
+ 1.0,
184
+ 1.0,
185
+ 1.0,
186
+ 1.0,
187
+ 1.0,
188
+ 1.0,
189
+ 1.0,
190
+ 1.0,
191
+ 1.0,
192
+ 1.0,
193
+ 1.0,
194
+ 1.0,
195
+ 1.0,
196
+ 1.0,
197
+ 1.0,
198
+ 1.0
199
+ ],
200
+ "type": "longrope"
201
+ },
202
+ "rope_theta": 10000.0,
203
+ "sliding_window": 262144,
204
+ "speech_lora": {
205
+ "dp": 0.01,
206
+ "layer": "((layers.*self_attn\\.(qkv|o)_proj)|(layers.*mlp\\.(gate_up|down)_proj))",
207
+ "lora_alpha": 640,
208
+ "r": 320
209
+ },
210
+ "tie_word_embeddings": true,
211
+ "torch_dtype": "bfloat16",
212
+ "transformers_version": "4.48.2",
213
+ "use_cache": true,
214
+ "vision_lora": {
215
+ "dp": 0.0,
216
+ "layer": "layers.*((self_attn\\.(qkv_proj|o_proj))|(mlp\\.(gate_up|down)_proj))",
217
+ "lora_alpha": 512,
218
+ "r": 256
219
+ },
220
+ "vocab_size": 200064
221
+ }
eval_before.json ADDED
@@ -0,0 +1,376 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "predictions_and_labels": [
3
+ {
4
+ "prediction": "2.5",
5
+ "label": "2.0"
6
+ },
7
+ {
8
+ "prediction": "2.0",
9
+ "label": "2.0"
10
+ },
11
+ {
12
+ "prediction": "4.0",
13
+ "label": "4.0"
14
+ },
15
+ {
16
+ "prediction": "4.0",
17
+ "label": "4.0"
18
+ },
19
+ {
20
+ "prediction": "3.0",
21
+ "label": "3.0"
22
+ },
23
+ {
24
+ "prediction": "3.0",
25
+ "label": "3.0"
26
+ },
27
+ {
28
+ "prediction": "2.5",
29
+ "label": "2.5"
30
+ },
31
+ {
32
+ "prediction": "4.0",
33
+ "label": "4.0"
34
+ },
35
+ {
36
+ "prediction": "3.0",
37
+ "label": "3.0"
38
+ },
39
+ {
40
+ "prediction": "4.0",
41
+ "label": "4.0"
42
+ },
43
+ {
44
+ "prediction": "3.0",
45
+ "label": "3.0"
46
+ },
47
+ {
48
+ "prediction": "4.0",
49
+ "label": "3.5"
50
+ },
51
+ {
52
+ "prediction": "4.0",
53
+ "label": "4.0"
54
+ },
55
+ {
56
+ "prediction": "4.0",
57
+ "label": "4.5"
58
+ },
59
+ {
60
+ "prediction": "4.0",
61
+ "label": "4.0"
62
+ },
63
+ {
64
+ "prediction": "4.0",
65
+ "label": "4.0"
66
+ },
67
+ {
68
+ "prediction": "3.0",
69
+ "label": "3.0"
70
+ },
71
+ {
72
+ "prediction": "3.0",
73
+ "label": "3.0"
74
+ },
75
+ {
76
+ "prediction": "3.5",
77
+ "label": "3.0"
78
+ },
79
+ {
80
+ "prediction": "3.0",
81
+ "label": "3.0"
82
+ },
83
+ {
84
+ "prediction": "4.0",
85
+ "label": "4.0"
86
+ },
87
+ {
88
+ "prediction": "3.5",
89
+ "label": "4.0"
90
+ },
91
+ {
92
+ "prediction": "3.0",
93
+ "label": "4.0"
94
+ },
95
+ {
96
+ "prediction": "3.0",
97
+ "label": "3.0"
98
+ },
99
+ {
100
+ "prediction": "4.0",
101
+ "label": "4.0"
102
+ },
103
+ {
104
+ "prediction": "3.0",
105
+ "label": "3.0"
106
+ },
107
+ {
108
+ "prediction": "4.5",
109
+ "label": "5.0"
110
+ },
111
+ {
112
+ "prediction": "3.0",
113
+ "label": "3.5"
114
+ },
115
+ {
116
+ "prediction": "3.0",
117
+ "label": "3.5"
118
+ },
119
+ {
120
+ "prediction": "3.5",
121
+ "label": "3.5"
122
+ },
123
+ {
124
+ "prediction": "4.0",
125
+ "label": "4.0"
126
+ },
127
+ {
128
+ "prediction": "3.5",
129
+ "label": "3.5"
130
+ },
131
+ {
132
+ "prediction": "4.0",
133
+ "label": "3.5"
134
+ },
135
+ {
136
+ "prediction": "4.0",
137
+ "label": "4.0"
138
+ },
139
+ {
140
+ "prediction": "4.0",
141
+ "label": "5.0"
142
+ },
143
+ {
144
+ "prediction": "4.0",
145
+ "label": "4.0"
146
+ },
147
+ {
148
+ "prediction": "4.0",
149
+ "label": "4.0"
150
+ },
151
+ {
152
+ "prediction": "4.0",
153
+ "label": "4.0"
154
+ },
155
+ {
156
+ "prediction": "4.0",
157
+ "label": "4.0"
158
+ },
159
+ {
160
+ "prediction": "4.5",
161
+ "label": "4.5"
162
+ },
163
+ {
164
+ "prediction": "4.0",
165
+ "label": "4.0"
166
+ },
167
+ {
168
+ "prediction": "3.0",
169
+ "label": "3.0"
170
+ },
171
+ {
172
+ "prediction": "3.0",
173
+ "label": "3.0"
174
+ },
175
+ {
176
+ "prediction": "3.0",
177
+ "label": "2.5"
178
+ },
179
+ {
180
+ "prediction": "4.0",
181
+ "label": "4.0"
182
+ },
183
+ {
184
+ "prediction": "4.0",
185
+ "label": "4.0"
186
+ },
187
+ {
188
+ "prediction": "4.0",
189
+ "label": "4.0"
190
+ },
191
+ {
192
+ "prediction": "4.0",
193
+ "label": "4.5"
194
+ },
195
+ {
196
+ "prediction": "3.5",
197
+ "label": "3.5"
198
+ },
199
+ {
200
+ "prediction": "3.5",
201
+ "label": "3.5"
202
+ },
203
+ {
204
+ "prediction": "4.5",
205
+ "label": "4.5"
206
+ },
207
+ {
208
+ "prediction": "4.0",
209
+ "label": "3.5"
210
+ },
211
+ {
212
+ "prediction": "3.0",
213
+ "label": "3.0"
214
+ },
215
+ {
216
+ "prediction": "3.0",
217
+ "label": "3.5"
218
+ },
219
+ {
220
+ "prediction": "4.0",
221
+ "label": "4.0"
222
+ },
223
+ {
224
+ "prediction": "4.0",
225
+ "label": "4.5"
226
+ },
227
+ {
228
+ "prediction": "3.5",
229
+ "label": "4.0"
230
+ },
231
+ {
232
+ "prediction": "3.5",
233
+ "label": "3.5"
234
+ },
235
+ {
236
+ "prediction": "3.0",
237
+ "label": "3.0"
238
+ },
239
+ {
240
+ "prediction": "3.5",
241
+ "label": "3.5"
242
+ },
243
+ {
244
+ "prediction": "4.0",
245
+ "label": "4.0"
246
+ },
247
+ {
248
+ "prediction": "4.0",
249
+ "label": "4.0"
250
+ },
251
+ {
252
+ "prediction": "4.0",
253
+ "label": "4.5"
254
+ },
255
+ {
256
+ "prediction": "2.5",
257
+ "label": "3.0"
258
+ },
259
+ {
260
+ "prediction": "3.5",
261
+ "label": "4.5"
262
+ },
263
+ {
264
+ "prediction": "3.0",
265
+ "label": "3.0"
266
+ },
267
+ {
268
+ "prediction": "3.0",
269
+ "label": "4.0"
270
+ },
271
+ {
272
+ "prediction": "4.0",
273
+ "label": "4.0"
274
+ },
275
+ {
276
+ "prediction": "3.5",
277
+ "label": "4.0"
278
+ },
279
+ {
280
+ "prediction": "3.0",
281
+ "label": "3.5"
282
+ },
283
+ {
284
+ "prediction": "3.0",
285
+ "label": "3.0"
286
+ },
287
+ {
288
+ "prediction": "4.5",
289
+ "label": "4.5"
290
+ },
291
+ {
292
+ "prediction": "3.5",
293
+ "label": "4.0"
294
+ },
295
+ {
296
+ "prediction": "3.0",
297
+ "label": "3.0"
298
+ },
299
+ {
300
+ "prediction": "3.0",
301
+ "label": "2.5"
302
+ },
303
+ {
304
+ "prediction": "2.5",
305
+ "label": "3.0"
306
+ },
307
+ {
308
+ "prediction": "3.0",
309
+ "label": "4.0"
310
+ },
311
+ {
312
+ "prediction": "3.5",
313
+ "label": "3.5"
314
+ },
315
+ {
316
+ "prediction": "2.5",
317
+ "label": "3.0"
318
+ },
319
+ {
320
+ "prediction": "3.0",
321
+ "label": "3.5"
322
+ },
323
+ {
324
+ "prediction": "4.0",
325
+ "label": "4.0"
326
+ },
327
+ {
328
+ "prediction": "4.0",
329
+ "label": "3.0"
330
+ },
331
+ {
332
+ "prediction": "4.0",
333
+ "label": "4.0"
334
+ },
335
+ {
336
+ "prediction": "4.0",
337
+ "label": "3.5"
338
+ },
339
+ {
340
+ "prediction": "4.0",
341
+ "label": "4.0"
342
+ },
343
+ {
344
+ "prediction": "4.0",
345
+ "label": "4.0"
346
+ },
347
+ {
348
+ "prediction": "4.0",
349
+ "label": "4.0"
350
+ },
351
+ {
352
+ "prediction": "4.0",
353
+ "label": "4.0"
354
+ },
355
+ {
356
+ "prediction": "3.5",
357
+ "label": "3.5"
358
+ },
359
+ {
360
+ "prediction": "4.0",
361
+ "label": "4.0"
362
+ },
363
+ {
364
+ "prediction": "2.5",
365
+ "label": "2.0"
366
+ },
367
+ {
368
+ "prediction": "4.0",
369
+ "label": "4.0"
370
+ }
371
+ ],
372
+ "accuracy": 0.9347826086956522,
373
+ "absolute_accuracy": 0.782608695652174,
374
+ "binary_accuracy": 0.8586956521739131,
375
+ "num_samples": 92
376
+ }
model-00001-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ce8ab1bf72acde4aae064b69fe9409c20e9292a8fe5c471944ccb6b18a93aa4
3
+ size 4997504848
model-00002-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:811b06ea0a89469ba99139651b56e8324964bf45f2eed79c11be98a9c89a9919
3
+ size 4952333128
model-00003-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:63ce7af9803c4cde3a5084a8027434cdffb72ca503d45bb39b056e60e694f881
3
+ size 1199389232
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
runs/Apr17_16-00-45_f39ce0987bd5/events.out.tfevents.1744905677.f39ce0987bd5.19044.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6374287ed997fd1faf21be260c7b9e6fd6440782af51c6c7fea4bcbf36ea5d20
3
+ size 11285
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36d89b3c5404ed7f3b58f0bda16bc5e85c5cfeb1cb4de98b38b8450575c78ed2
3
+ size 5432