InfoTokenizers
/

tokenizers

Model card Files Files and versions Community

codebyzeb commited on May 9

Commit

47b2f9e

verified ·

1 Parent(s): 0e739a7

Upload folder using huggingface_hub

Browse files

Files changed (2) hide show

fw57M_Surprisal_thresholdB_8064/tokenizer.json +259 -256
fw57M_Surprisal_thresholdB_8064/vocab.json +0 -0

fw57M_Surprisal_thresholdB_8064/tokenizer.json CHANGED Viewed

@@ -40,7 +40,10 @@
     ]
   },
   "pre_tokenizer": {
-    "type": "WhitespaceSplit"
   },
   "post_processor": {
     "type": "ByteLevel",
@@ -318,262 +321,262 @@
       "Ł": 255,
       "ł": 256,
       "Ń": 257,
-      "##ĉ": 258,
       "##z": 259,
-      "##Ó": 260,
-      "##,": 261,
-      "##Ý": 262,
-      "##M": 263,
-      "##i": 264,
-      "##a": 265,
-      "##s": 266,
-      "##Y": 267,
-      "##|": 268,
-      "##R": 269,
-      "##d": 270,
-      "##j": 271,
-      "##°": 272,
-      "##³": 273,
-      "##Î": 274,
-      "##ď": 275,
-      "##;": 276,
-      "##<": 277,
-      "##ā": 278,
-      "##'": 279,
-      "##Ĺ": 280,
-      "##Ç": 281,
-      "##%": 282,
-      "##Ď": 283,
-      "##]": 284,
-      "##å": 285,
-      "##ĸ": 286,
-      "##ĵ": 287,
-      "##w": 288,
-      "##¿": 289,
-      "##È": 290,
-      "##Ē": 291,
-      "##E": 292,
-      "##I": 293,
-      "##ğ": 294,
-      "##ê": 295,
-      "##Ľ": 296,
-      "##$": 297,
-      "##ĕ": 298,
-      "##ě": 299,
-      "##P": 300,
-      "##Þ": 301,
-      "##J": 302,
-      "##Č": 303,
-      "##¸": 304,
-      "##m": 305,
-      "##č": 306,
-      "##¦": 307,
-      "##Ø": 308,
-      "##Á": 309,
-      "##`": 310,
-      "##À": 311,
-      "##@": 312,
-      "##/": 313,
-      "##4": 314,
-      "##Û": 315,
-      "##Ñ": 316,
-      "##Ĥ": 317,
-      "##Â": 318,
-      "##Q": 319,
-      "##Ğ": 320,
-      "##~": 321,
-      "##×": 322,
-      "##©": 323,
-      "##è": 324,
-      "##ù": 325,
-      "##q": 326,
-      "##c": 327,
-      "##h": 328,
-      "##ú": 329,
-      "##Ġ": 330,
-      "##į": 331,
-      "##Ð": 332,
-      "##ã": 333,
-      "##.": 334,
-      "##[": 335,
-      "##k": 336,
-      "##}": 337,
-      "##Ė": 338,
-      "##Į": 339,
-      "##G": 340,
-      "##Ĭ": 341,
-      "##3": 342,
-      "##+": 343,
-      "##ķ": 344,
-      "##Ł": 345,
-      "##£": 346,
-      "##Ĝ": 347,
-      "##Ą": 348,
-      "##ė": 349,
-      "##K": 350,
-      "##_": 351,
-      "##õ": 352,
-      "##N": 353,
-      "##ï": 354,
-      "##7": 355,
-      "##ä": 356,
-      "##²": 357,
-      "##ă": 358,
-      "##í": 359,
-      "##ü": 360,
-      "##ľ": 361,
-      "##*": 362,
-      "##\"": 363,
-      "##f": 364,
-      "##ë": 365,
-      "##ī": 366,
-      "##Ĳ": 367,
-      "##¡": 368,
-      "##Ê": 369,
-      "##þ": 370,
-      "##8": 371,
-      "##>": 372,
-      "##Ċ": 373,
-      "##Ô": 374,
-      "##ø": 375,
-      "##n": 376,
-      "##İ": 377,
-      "##Ă": 378,
-      "##¯": 379,
-      "##ć": 380,
-      "##·": 381,
-      "##Ģ": 382,
-      "##Ĉ": 383,
-      "##ĝ": 384,
-      "##ħ": 385,
-      "##b": 386,
-      "##É": 387,
-      "##H": 388,
-      "##9": 389,
-      "##û": 390,
-      "##l": 391,
-      "##-": 392,
-      "##r": 393,
-      "##ó": 394,
-      "##ę": 395,
-      "##D": 396,
-      "##ª": 397,
-      "##ð": 398,
-      "##¬": 399,
-      "##v": 400,
-      "##5": 401,
-      "##Ì": 402,
-      "##Z": 403,
-      "##æ": 404,
-      "##Ù": 405,
-      "##ô": 406,
-      "##ł": 407,
-      "##e": 408,
-      "##S": 409,
-      "##Ļ": 410,
-      "##L": 411,
-      "##à": 412,
-      "##Ò": 413,
-      "##ġ": 414,
-      "##:": 415,
-      "##Ï": 416,
-      "##ą": 417,
-      "##Ë": 418,
-      "##Ĵ": 419,
-      "##g": 420,
-      "##Ć": 421,
-      "##ı": 422,
-      "##6": 423,
-      "##B": 424,
-      "##p": 425,
-      "###": 426,
-      "##¨": 427,
-      "##Ĩ": 428,
-      "##O": 429,
-      "##¹": 430,
-      "##¼": 431,
-      "##Ě": 432,
-      "##y": 433,
-      "##Ã": 434,
-      "##º": 435,
-      "##Ü": 436,
-      "##«": 437,
-      "##ĭ": 438,
-      "##ö": 439,
-      "##=": 440,
-      "##A": 441,
-      "##§": 442,
-      "##¾": 443,
-      "##ļ": 444,
-      "##±": 445,
-      "##Ę": 446,
-      "##Ķ": 447,
-      "##Ŀ": 448,
-      "##ß": 449,
-      "##X": 450,
-      "##ñ": 451,
-      "##ò": 452,
-      "##t": 453,
-      "##Ī": 454,
-      "##÷": 455,
-      "##ĺ": 456,
-      "##Í": 457,
-      "##T": 458,
-      "##Ö": 459,
-      "##o": 460,
-      "##1": 461,
-      "##½": 462,
-      "##)": 463,
-      "##»": 464,
-      "##á": 465,
-      "##¶": 466,
-      "##ý": 467,
-      "##ĩ": 468,
-      "##U": 469,
-      "##Ä": 470,
-      "##W": 471,
-      "##2": 472,
-      "##Ħ": 473,
-      "##¥": 474,
-      "##!": 475,
-      "##ċ": 476,
-      "##ĳ": 477,
-      "##\\": 478,
-      "##µ": 479,
-      "##ì": 480,
-      "##&": 481,
-      "##ÿ": 482,
-      "##ĥ": 483,
-      "##F": 484,
-      "##C": 485,
-      "##Æ": 486,
-      "##đ": 487,
-      "##®": 488,
-      "##Å": 489,
-      "##V": 490,
-      "##Ú": 491,
-      "##â": 492,
-      "##?": 493,
-      "##(": 494,
-      "##0": 495,
-      "##x": 496,
-      "##ģ": 497,
-      "##^": 498,
-      "##î": 499,
-      "##Đ": 500,
-      "##Ĕ": 501,
-      "##Ń": 502,
-      "##ŀ": 503,
-      "##ç": 504,
-      "##é": 505,
-      "##u": 506,
-      "##ē": 507,
-      "##´": 508,
-      "##Ā": 509,
-      "##¤": 510,
-      "##Õ": 511,
-      "##{": 512,
-      "##¢": 513,
       "<|unk|>": 514,
       "##ng": 515,
       "##�": 516,

     ]
   },
   "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
   },
   "post_processor": {
     "type": "ByteLevel",
       "Ł": 255,
       "ł": 256,
       "Ń": 257,
+      "##>": 258,
       "##z": 259,
+      "##Ï": 260,
+      "##ú": 261,
+      "##M": 262,
+      "##^": 263,
+      "##¢": 264,
+      "##T": 265,
+      "##.": 266,
+      "##6": 267,
+      "##:": 268,
+      "##Z": 269,
+      "##¯": 270,
+      "##È": 271,
+      "##q": 272,
+      "##ç": 273,
+      "##ò": 274,
+      "##9": 275,
+      "##ì": 276,
+      "##ě": 277,
+      "##k": 278,
+      "##u": 279,
+      "##$": 280,
+      "##ĥ": 281,
+      "##ē": 282,
+      "##-": 283,
+      "##Å": 284,
+      "##G": 285,
+      "##b": 286,
+      "##Ð": 287,
+      "##ğ": 288,
+      "##Â": 289,
+      "##č": 290,
+      "##÷": 291,
+      "##`": 292,
+      "##ĩ": 293,
+      "##Ù": 294,
+      "##ď": 295,
+      "##ä": 296,
+      "##¸": 297,
+      "##ê": 298,
+      "##¾": 299,
+      "##X": 300,
+      "##Ğ": 301,
+      "##+": 302,
+      "##Ü": 303,
+      "##ą": 304,
+      "##g": 305,
+      "##À": 306,
+      "##Ĉ": 307,
+      "##&": 308,
+      "##h": 309,
+      "##ñ": 310,
+      "##İ": 311,
+      "##%": 312,
+      "##ã": 313,
+      "##Í": 314,
+      "##Ď": 315,
+      "##H": 316,
+      "##ª": 317,
+      "##Ø": 318,
+      "##7": 319,
+      "##®": 320,
+      "##´": 321,
+      "##õ": 322,
+      "##¦": 323,
+      "##ı": 324,
+      "##¬": 325,
+      "##E": 326,
+      "##Ñ": 327,
+      "##3": 328,
+      "##é": 329,
+      "##ï": 330,
+      "##Ć": 331,
+      "##ħ": 332,
+      "##_": 333,
+      "##×": 334,
+      "##á": 335,
+      "##Ę": 336,
+      "##ļ": 337,
+      "##¿": 338,
+      "##¶": 339,
+      "##ă": 340,
+      "##ð": 341,
+      "##±": 342,
+      "##Ľ": 343,
+      "##R": 344,
+      "##Ö": 345,
+      "##Ĵ": 346,
+      "##ü": 347,
+      "##U": 348,
+      "##]": 349,
+      "##8": 350,
+      "##í": 351,
+      "##I": 352,
+      "##ø": 353,
+      "##F": 354,
+      "##æ": 355,
+      "##Ā": 356,
+      "##ĭ": 357,
+      "##m": 358,
+      "##è": 359,
+      "###": 360,
+      "##L": 361,
+      "##S": 362,
+      "##[": 363,
+      "##ë": 364,
+      "##i": 365,
+      "##Ĳ": 366,
+      "##c": 367,
+      "##(": 368,
+      "##Ġ": 369,
+      "##5": 370,
+      "##ċ": 371,
+      "##ó": 372,
+      "##s": 373,
+      "##å": 374,
+      "##»": 375,
+      "##~": 376,
+      "##Ń": 377,
+      "##į": 378,
+      "##C": 379,
+      "##p": 380,
+      "##*": 381,
+      "##@": 382,
+      "##ę": 383,
+      "##ė": 384,
+      "##Ĝ": 385,
+      "##à": 386,
+      "##V": 387,
+      "##Č": 388,
+      "##ö": 389,
+      "##?": 390,
+      "##Ħ": 391,
+      "##ī": 392,
+      "##Ĭ": 393,
+      "##Ē": 394,
+      "##e": 395,
+      "##Ú": 396,
+      "##Ò": 397,
+      "##ô": 398,
+      "##£": 399,
+      "##)": 400,
+      "##ł": 401,
+      "##Ĺ": 402,
+      "##{": 403,
+      "##°": 404,
+      "##x": 405,
+      "##N": 406,
+      "##Ä": 407,
+      "##ģ": 408,
+      "##!": 409,
+      "##µ": 410,
+      "##Ă": 411,
+      "##·": 412,
+      "##ÿ": 413,
+      "##v": 414,
+      "##J": 415,
+      "##a": 416,
+      "##³": 417,
+      "##Õ": 418,
+      "##0": 419,
+      "##û": 420,
+      "##4": 421,
+      "##Ī": 422,
+      "##Ķ": 423,
+      "##ć": 424,
+      "##w": 425,
+      "##©": 426,
+      "##ĸ": 427,
+      "##Ĕ": 428,
+      "##2": 429,
+      "##ý": 430,
+      "##É": 431,
+      "##n": 432,
+      "##Ĩ": 433,
+      "##ĳ": 434,
+      "##/": 435,
+      "##l": 436,
+      "##Û": 437,
+      "##Ã": 438,
+      "##=": 439,
+      "##ľ": 440,
+      "##r": 441,
+      "##K": 442,
+      "##¤": 443,
+      "##Đ": 444,
+      "##²": 445,
+      "##ġ": 446,
+      "##Y": 447,
+      "##ù": 448,
+      "##Ł": 449,
+      "##,": 450,
+      "##Ô": 451,
+      "##þ": 452,
+      "##º": 453,
+      "##P": 454,
+      "##j": 455,
+      "##§": 456,
+      "##¹": 457,
+      "##Ë": 458,
+      "##Á": 459,
+      "##â": 460,
+      "##'": 461,
+      "##¥": 462,
+      "##A": 463,
+      "##Ê": 464,
+      "##ā": 465,
+      "##ß": 466,
+      "##Ė": 467,
+      "##Į": 468,
+      "##Ģ": 469,
+      "##B": 470,
+      "##Ċ": 471,
+      "##}": 472,
+      "##î": 473,
+      "##Ě": 474,
+      "##f": 475,
+      "##Ç": 476,
+      "##<": 477,
+      "##½": 478,
+      "##¡": 479,
+      "##W": 480,
+      "##t": 481,
+      "##đ": 482,
+      "##Ŀ": 483,
+      "##1": 484,
+      "##d": 485,
+      "##Ì": 486,
+      "##Ý": 487,
+      "##Ą": 488,
+      "##o": 489,
+      "##y": 490,
+      "##\"": 491,
+      "##¨": 492,
+      "##«": 493,
+      "##ĝ": 494,
+      "##ķ": 495,
+      "##;": 496,
+      "##Q": 497,
+      "##O": 498,
+      "##ĺ": 499,
+      "##\\": 500,
+      "##Ó": 501,
+      "##ĵ": 502,
+      "##|": 503,
+      "##Î": 504,
+      "##Æ": 505,
+      "##¼": 506,
+      "##Ĥ": 507,
+      "##Þ": 508,
+      "##Ļ": 509,
+      "##ĕ": 510,
+      "##D": 511,
+      "##ĉ": 512,
+      "##ŀ": 513,
       "<|unk|>": 514,
       "##ng": 515,
       "##�": 516,

fw57M_Surprisal_thresholdB_8064/vocab.json CHANGED Viewed

The diff for this file is too large to render. See raw diff