InfoTokenizers
/

tokenizers

Model card Files Files and versions Community

codebyzeb commited on May 9

Commit

162d166

verified ·

1 Parent(s): 67cc952

Upload folder using huggingface_hub

Browse files

Files changed (2) hide show

fw57Mmulti_Entropy_thresholdB_32000/tokenizer.json +259 -256
fw57Mmulti_Entropy_thresholdB_32000/vocab.json +0 -0

fw57Mmulti_Entropy_thresholdB_32000/tokenizer.json CHANGED Viewed

@@ -40,7 +40,10 @@
     ]
   },
   "pre_tokenizer": {
-    "type": "WhitespaceSplit"
   },
   "post_processor": {
     "type": "ByteLevel",
@@ -318,262 +321,262 @@
       "Ł": 255,
       "ł": 256,
       "Ń": 257,
-      "##Z": 258,
-      "##s": 259,
-      "##L": 260,
-      "##à": 261,
-      "##O": 262,
-      "##t": 263,
-      "##ĸ": 264,
-      "##+": 265,
-      "##Đ": 266,
-      "##×": 267,
-      "##,": 268,
-      "##Á": 269,
-      "##ě": 270,
-      "##ķ": 271,
-      "##K": 272,
-      "##n": 273,
-      "##đ": 274,
-      "##İ": 275,
-      "##¢": 276,
-      "##Õ": 277,
-      "##í": 278,
-      "##Ą": 279,
-      "##Ý": 280,
-      "##¼": 281,
-      "##Ģ": 282,
-      "##¶": 283,
-      "##X": 284,
-      "##ā": 285,
-      "##ē": 286,
-      "##!": 287,
-      "##¸": 288,
-      "##Å": 289,
-      "##H": 290,
-      "##¾": 291,
-      "##Į": 292,
-      "##j": 293,
-      "##ß": 294,
-      "##Y": 295,
-      "##Î": 296,
-      "##Ł": 297,
-      "##Ļ": 298,
-      "##±": 299,
-      "##R": 300,
-      "##i": 301,
-      "##\"": 302,
-      "##;": 303,
-      "##0": 304,
-      "##o": 305,
-      "##Ô": 306,
-      "##ì": 307,
-      "##ð": 308,
-      "##?": 309,
-      "##Č": 310,
-      "##.": 311,
-      "##ĝ": 312,
-      "##*": 313,
-      "##î": 314,
-      "##Ğ": 315,
-      "##Ġ": 316,
-      "##A": 317,
-      "##ĳ": 318,
-      "##¥": 319,
-      "##C": 320,
-      "##Æ": 321,
-      "##µ": 322,
-      "##ë": 323,
-      "##f": 324,
-      "##«": 325,
-      "##Q": 326,
-      "##Ú": 327,
-      "##Â": 328,
-      "##ğ": 329,
-      "##u": 330,
-      "##J": 331,
-      "##_": 332,
-      "##ċ": 333,
-      "##å": 334,
-      "##{": 335,
-      "##V": 336,
-      "##Ħ": 337,
-      "##E": 338,
-      "##Ĉ": 339,
-      "##ñ": 340,
-      "##į": 341,
-      "##7": 342,
-      "##5": 343,
-      "##Ķ": 344,
-      "##Ò": 345,
-      "##%": 346,
-      "##Í": 347,
-      "##q": 348,
-      "##č": 349,
-      "##2": 350,
-      "##a": 351,
-      "##ł": 352,
-      "##²": 353,
-      "##£": 354,
-      "##>": 355,
-      "##¿": 356,
-      "##h": 357,
-      "##Ĭ": 358,
-      "##Ċ": 359,
-      "##Þ": 360,
-      "##û": 361,
-      "##w": 362,
-      "##F": 363,
-      "##|": 364,
-      "##Ó": 365,
-      "##Ĕ": 366,
       "##6": 367,
-      "##ď": 368,
-      "##Ę": 369,
-      "##d": 370,
-      "##9": 371,
-      "##/": 372,
-      "##Ě": 373,
-      "##-": 374,
-      "##p": 375,
-      "##ĵ": 376,
-      "##ç": 377,
-      "##b": 378,
-      "##ļ": 379,
-      "##´": 380,
-      "##Ŀ": 381,
-      "##ĉ": 382,
-      "##º": 383,
-      "##c": 384,
-      "##È": 385,
-      "##è": 386,
-      "##Ê": 387,
-      "##^": 388,
-      "##ú": 389,
-      "##â": 390,
-      "##ò": 391,
-      "##Ľ": 392,
-      "##»": 393,
-      "##Ø": 394,
-      "##ö": 395,
-      "##Ã": 396,
-      "##á": 397,
-      "##é": 398,
-      "##ı": 399,
-      "##É": 400,
-      "##Û": 401,
-      "##Ī": 402,
-      "##§": 403,
-      "##ø": 404,
-      "##Ă": 405,
-      "##ŀ": 406,
-      "##Ĥ": 407,
-      "##m": 408,
-      "##ġ": 409,
-      "##Ď": 410,
-      "##`": 411,
-      "##Ć": 412,
-      "##)": 413,
-      "##æ": 414,
-      "##ħ": 415,
-      "##Ä": 416,
-      "###": 417,
-      "##Ð": 418,
-      "##ê": 419,
-      "##x": 420,
-      "##U": 421,
-      "##ã": 422,
-      "##z": 423,
-      "##ė": 424,
-      "##e": 425,
-      "##¹": 426,
-      "##Ė": 427,
-      "##ĺ": 428,
-      "##1": 429,
-      "##Ù": 430,
-      "##M": 431,
-      "##Ĵ": 432,
-      "##Ĝ": 433,
-      "##½": 434,
-      "##G": 435,
-      "##<": 436,
-      "##Ē": 437,
-      "##õ": 438,
-      "##T": 439,
-      "##l": 440,
-      "##&": 441,
-      "##}": 442,
-      "##D": 443,
-      "##ä": 444,
-      "##ý": 445,
-      "##:": 446,
-      "##©": 447,
-      "##k": 448,
-      "##¨": 449,
-      "##8": 450,
-      "##ô": 451,
-      "##ĩ": 452,
-      "##P": 453,
-      "##ĭ": 454,
-      "##®": 455,
-      "##þ": 456,
-      "##ľ": 457,
-      "##ć": 458,
-      "##I": 459,
-      "##v": 460,
-      "##4": 461,
-      "##ª": 462,
-      "##r": 463,
-      "##·": 464,
-      "##W": 465,
-      "##[": 466,
-      "##B": 467,
-      "##Ü": 468,
-      "##~": 469,
-      "##3": 470,
-      "##ÿ": 471,
-      "##@": 472,
-      "##ă": 473,
-      "##Ĩ": 474,
-      "##ģ": 475,
-      "##¯": 476,
-      "##Ń": 477,
-      "##¦": 478,
-      "##Ì": 479,
-      "##Ç": 480,
-      "##Ë": 481,
-      "##ī": 482,
-      "##ù": 483,
-      "##÷": 484,
-      "##³": 485,
-      "##g": 486,
-      "##ĕ": 487,
-      "##ą": 488,
-      "##Ö": 489,
-      "##Ĳ": 490,
-      "##¤": 491,
-      "##ü": 492,
-      "##À": 493,
-      "##\\": 494,
-      "##=": 495,
-      "##ï": 496,
-      "##ę": 497,
-      "##y": 498,
-      "##$": 499,
-      "##N": 500,
-      "##Ï": 501,
-      "##¡": 502,
-      "##Ñ": 503,
-      "##°": 504,
-      "##Ā": 505,
-      "##¬": 506,
-      "##ĥ": 507,
-      "##(": 508,
-      "##S": 509,
-      "##]": 510,
-      "##Ĺ": 511,
-      "##ó": 512,
-      "##'": 513,
       "<|unk|>": 514,
       "##�": 515,
       "sa": 516,

     ]
   },
   "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
   },
   "post_processor": {
     "type": "ByteLevel",
       "Ł": 255,
       "ł": 256,
       "Ń": 257,
+      "##)": 258,
+      "##ĝ": 259,
+      "##¶": 260,
+      "##|": 261,
+      "##Ē": 262,
+      "##Ě": 263,
+      "##`": 264,
+      "##ç": 265,
+      "##÷": 266,
+      "##p": 267,
+      "##=": 268,
+      "##2": 269,
+      "##U": 270,
+      "##s": 271,
+      "##¢": 272,
+      "##Í": 273,
+      "##Ê": 274,
+      "##ŀ": 275,
+      "##ú": 276,
+      "##û": 277,
+      "##ĳ": 278,
+      "##3": 279,
+      "##k": 280,
+      "##1": 281,
+      "##o": 282,
+      "##T": 283,
+      "##é": 284,
+      "##[": 285,
+      "##ċ": 286,
+      "##-": 287,
+      "##ħ": 288,
+      "##,": 289,
+      "##Ë": 290,
+      "##ĺ": 291,
+      "##I": 292,
+      "##đ": 293,
+      "##Ü": 294,
+      "##Ļ": 295,
+      "##ß": 296,
+      "##;": 297,
+      "##Ñ": 298,
+      "##R": 299,
+      "##@": 300,
+      "##\\": 301,
+      "##r": 302,
+      "##]": 303,
+      "##»": 304,
+      "##Ð": 305,
+      "##M": 306,
+      "##â": 307,
+      "##Ô": 308,
+      "##Õ": 309,
+      "##Đ": 310,
+      "##ę": 311,
+      "##ì": 312,
+      "##ð": 313,
+      "##¸": 314,
+      "##ı": 315,
+      "##_": 316,
+      "##4": 317,
+      "##ļ": 318,
+      "##Ğ": 319,
+      "##<": 320,
+      "##n": 321,
+      "##É": 322,
+      "##'": 323,
+      "##ò": 324,
+      "##0": 325,
+      "##C": 326,
+      "##ď": 327,
+      "##a": 328,
+      "##Ą": 329,
+      "##Ĭ": 330,
+      "##Æ": 331,
+      "##Ā": 332,
+      "##ģ": 333,
+      "##q": 334,
+      "##N": 335,
+      "##·": 336,
+      "##¬": 337,
+      "##Ý": 338,
+      "##½": 339,
+      "##Ď": 340,
+      "##ė": 341,
+      "##į": 342,
+      "##¤": 343,
+      "##Ĵ": 344,
+      "##Q": 345,
+      "##Ò": 346,
+      "##Ķ": 347,
+      "##L": 348,
+      "##ĭ": 349,
+      "##Ĉ": 350,
+      "##}": 351,
+      "##°": 352,
+      "##~": 353,
+      "##ĉ": 354,
+      "##^": 355,
+      "##!": 356,
+      "##ĕ": 357,
+      "##e": 358,
+      "##ä": 359,
+      "##þ": 360,
+      "##ö": 361,
+      "##Ć": 362,
+      "##Ċ": 363,
+      "##X": 364,
+      "##«": 365,
+      "##ľ": 366,
       "##6": 367,
+      "##l": 368,
+      "##à": 369,
+      "##Ń": 370,
+      "##Ú": 371,
+      "##5": 372,
+      "##Č": 373,
+      "##Ę": 374,
+      "##¡": 375,
+      "##W": 376,
+      "##ī": 377,
+      "##+": 378,
+      "##>": 379,
+      "##Ĝ": 380,
+      "##ł": 381,
+      "##®": 382,
+      "##%": 383,
+      "##:": 384,
+      "##&": 385,
+      "##Ç": 386,
+      "##ĸ": 387,
+      "##/": 388,
+      "##Ã": 389,
+      "##á": 390,
+      "##c": 391,
+      "##ē": 392,
+      "##Ï": 393,
+      "##¹": 394,
+      "##O": 395,
+      "##Ó": 396,
+      "##ñ": 397,
+      "##Ī": 398,
+      "##d": 399,
+      "##ù": 400,
+      "##Ă": 401,
+      "##æ": 402,
+      "##¾": 403,
+      "##ó": 404,
+      "##õ": 405,
+      "##ü": 406,
+      "##D": 407,
+      "##¯": 408,
+      "##Ġ": 409,
+      "##G": 410,
+      "##v": 411,
+      "##£": 412,
+      "##è": 413,
+      "##Ø": 414,
+      "##ý": 415,
+      "##ĩ": 416,
+      "##*": 417,
+      "##{": 418,
+      "##Â": 419,
+      "##h": 420,
+      "##w": 421,
+      "##¿": 422,
+      "##B": 423,
+      "##Á": 424,
+      "##t": 425,
+      "##ô": 426,
+      "###": 427,
+      "##?": 428,
+      "##¨": 429,
+      "##Ä": 430,
+      "##8": 431,
+      "##P": 432,
+      "##Û": 433,
+      "##ÿ": 434,
+      "##ā": 435,
+      "##Y": 436,
+      "##©": 437,
+      "##ě": 438,
+      "##Ì": 439,
+      "##ĥ": 440,
+      "##Ĥ": 441,
+      "##º": 442,
+      "##y": 443,
+      "##²": 444,
+      "##µ": 445,
+      "##E": 446,
+      "##¦": 447,
+      "##´": 448,
+      "##F": 449,
+      "##Ĕ": 450,
+      "##İ": 451,
+      "##Ľ": 452,
+      "##î": 453,
+      "##g": 454,
+      "##ª": 455,
+      "##¼": 456,
+      "##7": 457,
+      "##×": 458,
+      "##Å": 459,
+      "##b": 460,
+      "##§": 461,
+      "##ê": 462,
+      "##f": 463,
+      "##Ö": 464,
+      "##å": 465,
+      "##Z": 466,
+      "##ă": 467,
+      "##H": 468,
+      "##ġ": 469,
+      "##Ĩ": 470,
+      "##x": 471,
+      "##í": 472,
+      "##³": 473,
+      "##9": 474,
+      "##Þ": 475,
+      "##Ħ": 476,
+      "##J": 477,
+      "##Ł": 478,
+      "##Ĳ": 479,
+      "##ø": 480,
+      "##z": 481,
+      "##¥": 482,
+      "##j": 483,
+      "##È": 484,
+      "##č": 485,
+      "##ķ": 486,
+      "##V": 487,
+      "##Ù": 488,
+      "##S": 489,
+      "##$": 490,
+      "##ć": 491,
+      "##m": 492,
+      "##(": 493,
+      "##ï": 494,
+      "##Į": 495,
+      "##.": 496,
+      "##Ģ": 497,
+      "##i": 498,
+      "##Ŀ": 499,
+      "##Ĺ": 500,
+      "##A": 501,
+      "##±": 502,
+      "##ğ": 503,
+      "##Ė": 504,
+      "##u": 505,
+      "##Î": 506,
+      "##À": 507,
+      "##ë": 508,
+      "##ã": 509,
+      "##K": 510,
+      "##\"": 511,
+      "##ą": 512,
+      "##ĵ": 513,
       "<|unk|>": 514,
       "##�": 515,
       "sa": 516,

fw57Mmulti_Entropy_thresholdB_32000/vocab.json CHANGED Viewed

The diff for this file is too large to render. See raw diff