InfoTokenizers
/

tokenizers

Model card Files Files and versions Community

codebyzeb commited on May 8

Commit

62a0567

verified ·

1 Parent(s): f9c3417

Upload folder using huggingface_hub

Browse files

Files changed (2) hide show

fw57Mmulti_Entropy_thresholdB_32000/tokenizer.json +253 -253
fw57Mmulti_Entropy_thresholdB_32000/vocab.json +0 -0

fw57Mmulti_Entropy_thresholdB_32000/tokenizer.json CHANGED Viewed

@@ -318,262 +318,262 @@
       "Ł": 255,
       "ł": 256,
       "Ń": 257,
-      "##q": 258,
-      "##¡": 259,
-      "##ă": 260,
-      "##W": 261,
-      "##S": 262,
-      "##Ď": 263,
-      "##Ė": 264,
-      "##Ĝ": 265,
-      "##y": 266,
-      "##J": 267,
-      "###": 268,
-      "##I": 269,
-      "##Ø": 270,
-      "##N": 271,
-      "##Ò": 272,
-      "##ý": 273,
-      "##Y": 274,
-      "##¦": 275,
-      "##\\": 276,
-      "##È": 277,
-      "##ğ": 278,
-      "##¨": 279,
-      "##>": 280,
-      "##Á": 281,
-      "##Î": 282,
-      "##µ": 283,
-      "##;": 284,
-      "##¹": 285,
-      "##Ĳ": 286,
-      "##ĳ": 287,
-      "##±": 288,
-      "##M": 289,
-      "##°": 290,
-      "##á": 291,
-      "##<": 292,
-      "##é": 293,
-      "##Ę": 294,
-      "##ĭ": 295,
-      "##Ķ": 296,
-      "##Ġ": 297,
-      "##H": 298,
-      "##Ě": 299,
-      "##ę": 300,
-      "##7": 301,
-      "##¬": 302,
-      "##đ": 303,
-      "##@": 304,
-      "##6": 305,
-      "##|": 306,
-      "##Ì": 307,
-      "##ä": 308,
-      "##ć": 309,
-      "##=": 310,
-      "##'": 311,
-      "##û": 312,
-      "##ī": 313,
-      "##Û": 314,
-      "##Ĕ": 315,
-      "##.": 316,
       "##A": 317,
-      "##Ī": 318,
-      "##ě": 319,
-      "##£": 320,
-      "##ü": 321,
-      "##å": 322,
-      "##%": 323,
-      "##ò": 324,
-      "##Ŀ": 325,
-      "##ė": 326,
-      "##ñ": 327,
-      "##«": 328,
-      "##ē": 329,
-      "##ĸ": 330,
-      "##t": 331,
-      "##3": 332,
-      "##l": 333,
-      "##{": 334,
-      "##L": 335,
-      "##-": 336,
-      "##s": 337,
-      "##Ý": 338,
-      "##Ĥ": 339,
-      "##Å": 340,
-      "##¸": 341,
-      "##2": 342,
-      "##ª": 343,
-      "##Ð": 344,
-      "##ģ": 345,
-      "##æ": 346,
-      "##h": 347,
-      "##`": 348,
-      "##ġ": 349,
-      "##*": 350,
-      "##®": 351,
-      "##ĉ": 352,
-      "##Į": 353,
-      "##½": 354,
-      "##Ñ": 355,
-      "##í": 356,
-      "##5": 357,
-      "##î": 358,
-      "##\"": 359,
-      "##Ó": 360,
-      "##i": 361,
-      "##]": 362,
-      "##Ħ": 363,
-      "##Ĺ": 364,
-      "##à": 365,
-      "##c": 366,
-      "##×": 367,
-      "##Ú": 368,
-      "##ļ": 369,
-      "##§": 370,
-      "##ı": 371,
-      "##º": 372,
-      "##Þ": 373,
-      "##Đ": 374,
-      "##!": 375,
-      "##Ĩ": 376,
-      "##ï": 377,
-      "##ì": 378,
-      "##ô": 379,
-      "##Č": 380,
-      "##ö": 381,
-      "##Í": 382,
-      "##ħ": 383,
-      "##Ù": 384,
-      "##_": 385,
-      "##8": 386,
-      "##ĺ": 387,
-      "##u": 388,
-      "##Ç": 389,
-      "##Ĭ": 390,
-      "##d": 391,
-      "##ð": 392,
-      "##Ö": 393,
-      "##n": 394,
-      "##?": 395,
-      "##Ń": 396,
-      "##¼": 397,
-      "##B": 398,
-      "##³": 399,
-      "##ľ": 400,
-      "##¥": 401,
-      "##À": 402,
-      "##p": 403,
-      "##Ë": 404,
-      "##F": 405,
-      "##Ą": 406,
-      "##Ĉ": 407,
-      "##Ē": 408,
-      "##ó": 409,
-      "##Ā": 410,
-      "##a": 411,
-      "##X": 412,
-      "##Ć": 413,
-      "##~": 414,
-      "##ø": 415,
-      "##f": 416,
-      "##ĩ": 417,
-      "##©": 418,
-      "##D": 419,
-      "##&": 420,
-      "##V": 421,
-      "##^": 422,
-      "##0": 423,
-      "##1": 424,
-      "##j": 425,
-      "##Ô": 426,
-      "##į": 427,
-      "##P": 428,
-      "##w": 429,
-      "##Ģ": 430,
-      "##g": 431,
       "##Ĵ": 432,
-      "##ù": 433,
-      "##č": 434,
-      "##ĥ": 435,
-      "##ŀ": 436,
-      "##Ê": 437,
-      "##[": 438,
-      "##)": 439,
-      "##O": 440,
-      "##ā": 441,
-      "##Ã": 442,
-      "##ÿ": 443,
-      "##ł": 444,
-      "##e": 445,
-      "##x": 446,
-      "##ĝ": 447,
-      "##Ċ": 448,
-      "##´": 449,
-      "##Q": 450,
-      "##m": 451,
-      "##ĕ": 452,
-      "##Z": 453,
-      "##G": 454,
-      "##¾": 455,
-      "##ã": 456,
-      "##ë": 457,
-      "##ċ": 458,
-      "##Õ": 459,
       "##v": 460,
-      "##è": 461,
-      "##z": 462,
-      "##Â": 463,
-      "##÷": 464,
-      "##Ü": 465,
-      "##¤": 466,
-      "##Æ": 467,
-      "##:": 468,
-      "##b": 469,
-      "##Ï": 470,
-      "##o": 471,
-      "##r": 472,
-      "##õ": 473,
-      "##¶": 474,
-      "##Ļ": 475,
-      "##}": 476,
-      "##¿": 477,
-      "##T": 478,
-      "##²": 479,
-      "##E": 480,
-      "##k": 481,
-      "##9": 482,
-      "##ç": 483,
-      "##(": 484,
-      "##¢": 485,
-      "##ď": 486,
-      "##Ł": 487,
-      "##U": 488,
-      "##ĵ": 489,
-      "##C": 490,
-      "##·": 491,
-      "##4": 492,
-      "##þ": 493,
-      "##/": 494,
-      "##»": 495,
-      "##Ä": 496,
-      "##ß": 497,
-      "##ą": 498,
-      "##R": 499,
-      "##ú": 500,
-      "##İ": 501,
-      "##ê": 502,
-      "##Ľ": 503,
-      "##â": 504,
-      "##K": 505,
-      "##ķ": 506,
-      "##$": 507,
-      "##+": 508,
-      "##,": 509,
-      "##Ă": 510,
-      "##¯": 511,
-      "##É": 512,
-      "##Ğ": 513,
       "<|unk|>": 514,
       "##�": 515,
       "sa": 516,

       "Ł": 255,
       "ł": 256,
       "Ń": 257,
+      "##Z": 258,
+      "##s": 259,
+      "##L": 260,
+      "##à": 261,
+      "##O": 262,
+      "##t": 263,
+      "##ĸ": 264,
+      "##+": 265,
+      "##Đ": 266,
+      "##×": 267,
+      "##,": 268,
+      "##Á": 269,
+      "##ě": 270,
+      "##ķ": 271,
+      "##K": 272,
+      "##n": 273,
+      "##đ": 274,
+      "##İ": 275,
+      "##¢": 276,
+      "##Õ": 277,
+      "##í": 278,
+      "##Ą": 279,
+      "##Ý": 280,
+      "##¼": 281,
+      "##Ģ": 282,
+      "##¶": 283,
+      "##X": 284,
+      "##ā": 285,
+      "##ē": 286,
+      "##!": 287,
+      "##¸": 288,
+      "##Å": 289,
+      "##H": 290,
+      "##¾": 291,
+      "##Į": 292,
+      "##j": 293,
+      "##ß": 294,
+      "##Y": 295,
+      "##Î": 296,
+      "##Ł": 297,
+      "##Ļ": 298,
+      "##±": 299,
+      "##R": 300,
+      "##i": 301,
+      "##\"": 302,
+      "##;": 303,
+      "##0": 304,
+      "##o": 305,
+      "##Ô": 306,
+      "##ì": 307,
+      "##ð": 308,
+      "##?": 309,
+      "##Č": 310,
+      "##.": 311,
+      "##ĝ": 312,
+      "##*": 313,
+      "##î": 314,
+      "##Ğ": 315,
+      "##Ġ": 316,
       "##A": 317,
+      "##ĳ": 318,
+      "##¥": 319,
+      "##C": 320,
+      "##Æ": 321,
+      "##µ": 322,
+      "##ë": 323,
+      "##f": 324,
+      "##«": 325,
+      "##Q": 326,
+      "##Ú": 327,
+      "##Â": 328,
+      "##ğ": 329,
+      "##u": 330,
+      "##J": 331,
+      "##_": 332,
+      "##ċ": 333,
+      "##å": 334,
+      "##{": 335,
+      "##V": 336,
+      "##Ħ": 337,
+      "##E": 338,
+      "##Ĉ": 339,
+      "##ñ": 340,
+      "##į": 341,
+      "##7": 342,
+      "##5": 343,
+      "##Ķ": 344,
+      "##Ò": 345,
+      "##%": 346,
+      "##Í": 347,
+      "##q": 348,
+      "##č": 349,
+      "##2": 350,
+      "##a": 351,
+      "##ł": 352,
+      "##²": 353,
+      "##£": 354,
+      "##>": 355,
+      "##¿": 356,
+      "##h": 357,
+      "##Ĭ": 358,
+      "##Ċ": 359,
+      "##Þ": 360,
+      "##û": 361,
+      "##w": 362,
+      "##F": 363,
+      "##|": 364,
+      "##Ó": 365,
+      "##Ĕ": 366,
+      "##6": 367,
+      "##ď": 368,
+      "##Ę": 369,
+      "##d": 370,
+      "##9": 371,
+      "##/": 372,
+      "##Ě": 373,
+      "##-": 374,
+      "##p": 375,
+      "##ĵ": 376,
+      "##ç": 377,
+      "##b": 378,
+      "##ļ": 379,
+      "##´": 380,
+      "##Ŀ": 381,
+      "##ĉ": 382,
+      "##º": 383,
+      "##c": 384,
+      "##È": 385,
+      "##è": 386,
+      "##Ê": 387,
+      "##^": 388,
+      "##ú": 389,
+      "##â": 390,
+      "##ò": 391,
+      "##Ľ": 392,
+      "##»": 393,
+      "##Ø": 394,
+      "##ö": 395,
+      "##Ã": 396,
+      "##á": 397,
+      "##é": 398,
+      "##ı": 399,
+      "##É": 400,
+      "##Û": 401,
+      "##Ī": 402,
+      "##§": 403,
+      "##ø": 404,
+      "##Ă": 405,
+      "##ŀ": 406,
+      "##Ĥ": 407,
+      "##m": 408,
+      "##ġ": 409,
+      "##Ď": 410,
+      "##`": 411,
+      "##Ć": 412,
+      "##)": 413,
+      "##æ": 414,
+      "##ħ": 415,
+      "##Ä": 416,
+      "###": 417,
+      "##Ð": 418,
+      "##ê": 419,
+      "##x": 420,
+      "##U": 421,
+      "##ã": 422,
+      "##z": 423,
+      "##ė": 424,
+      "##e": 425,
+      "##¹": 426,
+      "##Ė": 427,
+      "##ĺ": 428,
+      "##1": 429,
+      "##Ù": 430,
+      "##M": 431,
       "##Ĵ": 432,
+      "##Ĝ": 433,
+      "##½": 434,
+      "##G": 435,
+      "##<": 436,
+      "##Ē": 437,
+      "##õ": 438,
+      "##T": 439,
+      "##l": 440,
+      "##&": 441,
+      "##}": 442,
+      "##D": 443,
+      "##ä": 444,
+      "##ý": 445,
+      "##:": 446,
+      "##©": 447,
+      "##k": 448,
+      "##¨": 449,
+      "##8": 450,
+      "##ô": 451,
+      "##ĩ": 452,
+      "##P": 453,
+      "##ĭ": 454,
+      "##®": 455,
+      "##þ": 456,
+      "##ľ": 457,
+      "##ć": 458,
+      "##I": 459,
       "##v": 460,
+      "##4": 461,
+      "##ª": 462,
+      "##r": 463,
+      "##·": 464,
+      "##W": 465,
+      "##[": 466,
+      "##B": 467,
+      "##Ü": 468,
+      "##~": 469,
+      "##3": 470,
+      "##ÿ": 471,
+      "##@": 472,
+      "##ă": 473,
+      "##Ĩ": 474,
+      "##ģ": 475,
+      "##¯": 476,
+      "##Ń": 477,
+      "##¦": 478,
+      "##Ì": 479,
+      "##Ç": 480,
+      "##Ë": 481,
+      "##ī": 482,
+      "##ù": 483,
+      "##÷": 484,
+      "##³": 485,
+      "##g": 486,
+      "##ĕ": 487,
+      "##ą": 488,
+      "##Ö": 489,
+      "##Ĳ": 490,
+      "##¤": 491,
+      "##ü": 492,
+      "##À": 493,
+      "##\\": 494,
+      "##=": 495,
+      "##ï": 496,
+      "##ę": 497,
+      "##y": 498,
+      "##$": 499,
+      "##N": 500,
+      "##Ï": 501,
+      "##¡": 502,
+      "##Ñ": 503,
+      "##°": 504,
+      "##Ā": 505,
+      "##¬": 506,
+      "##ĥ": 507,
+      "##(": 508,
+      "##S": 509,
+      "##]": 510,
+      "##Ĺ": 511,
+      "##ó": 512,
+      "##'": 513,
       "<|unk|>": 514,
       "##�": 515,
       "sa": 516,

fw57Mmulti_Entropy_thresholdB_32000/vocab.json CHANGED Viewed

The diff for this file is too large to render. See raw diff