xiongwang commited on
Commit
67f8902
·
verified ·
1 Parent(s): c1169d7

Upload 2 files

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +9 -2
  2. tokenizer_config.json +18 -3
special_tokens_map.json CHANGED
@@ -27,5 +27,12 @@
27
  "normalized": false,
28
  "rstrip": false,
29
  "single_word": false
30
- }
31
- }
 
 
 
 
 
 
 
 
27
  "normalized": false,
28
  "rstrip": false,
29
  "single_word": false
30
+ },
31
+ "image_token": "<|IMAGE|>",
32
+ "audio_token": "<|AUDIO|>",
33
+ "video_token": "<|VIDEO|>",
34
+ "vision_bos_token": "<|vision_bos|>",
35
+ "vision_eos_token": "<|vision_eos|>",
36
+ "audio_bos_token": "<|audio_bos|>",
37
+ "audio_eos_token": "<|audio_eos|>"
38
+ }
tokenizer_config.json CHANGED
@@ -198,11 +198,26 @@
198
  "clean_up_tokenization_spaces": false,
199
  "eos_token": "<|im_end|>",
200
  "errors": "replace",
201
- "extra_special_tokens": {},
 
 
 
 
 
 
 
 
202
  "model_max_length": 32768,
203
  "pad_token": "<|endoftext|>",
204
  "processor_class": "Qwen2_5OmniProcessor",
205
  "split_special_tokens": false,
206
  "tokenizer_class": "Qwen2Tokenizer",
207
- "unk_token": null
208
- }
 
 
 
 
 
 
 
 
198
  "clean_up_tokenization_spaces": false,
199
  "eos_token": "<|im_end|>",
200
  "errors": "replace",
201
+ "extra_special_tokens": {
202
+ "image_token": "<|IMAGE|>",
203
+ "audio_token": "<|AUDIO|>",
204
+ "video_token": "<|VIDEO|>",
205
+ "vision_bos_token": "<|vision_bos|>",
206
+ "vision_eos_token": "<|vision_eos|>",
207
+ "audio_bos_token": "<|audio_bos|>",
208
+ "audio_eos_token": "<|audio_eos|>"
209
+ },
210
  "model_max_length": 32768,
211
  "pad_token": "<|endoftext|>",
212
  "processor_class": "Qwen2_5OmniProcessor",
213
  "split_special_tokens": false,
214
  "tokenizer_class": "Qwen2Tokenizer",
215
+ "unk_token": null,
216
+ "image_token": "<|IMAGE|>",
217
+ "audio_token": "<|AUDIO|>",
218
+ "video_token": "<|VIDEO|>",
219
+ "vision_bos_token": "<|vision_bos|>",
220
+ "vision_eos_token": "<|vision_eos|>",
221
+ "audio_bos_token": "<|audio_bos|>",
222
+ "audio_eos_token": "<|audio_eos|>"
223
+ }