InfoTokenizers
/

tokenizers

Model card Files Files and versions Community

codebyzeb commited on May 5

Commit

e649e4e

verified ·

1 Parent(s): 7b7b585

Upload folder using huggingface_hub

Browse files

Files changed (2) hide show

fw57M_Entropy_thresholdB_16000/tokenizer.json +254 -254
fw57M_Entropy_thresholdB_16000/vocab.json +0 -0

fw57M_Entropy_thresholdB_16000/tokenizer.json CHANGED Viewed

@@ -318,262 +318,262 @@
       "Ł": 255,
       "ł": 256,
       "Ń": 257,
-      "##=": 258,
-      "##©": 259,
-      "##w": 260,
-      "##Ė": 261,
-      "###": 262,
-      "##Ë": 263,
-      "##_": 264,
-      "##Å": 265,
-      "##½": 266,
-      "##Ì": 267,
-      "##9": 268,
-      "##{": 269,
-      "##W": 270,
-      "##m": 271,
-      "##y": 272,
-      "##Ê": 273,
-      "##Â": 274,
-      "##4": 275,
-      "##7": 276,
-      "##Ó": 277,
-      "##ë": 278,
-      "##i": 279,
-      "##Ð": 280,
-      "##Đ": 281,
-      "##ļ": 282,
-      "##$": 283,
-      "##g": 284,
-      "##ĺ": 285,
-      "##´": 286,
-      "##r": 287,
-      "##Ď": 288,
-      "##ö": 289,
-      "##ð": 290,
-      "##¿": 291,
-      "##E": 292,
-      "##u": 293,
-      "##:": 294,
-      "##ĉ": 295,
-      "##Ù": 296,
-      "##ĭ": 297,
-      "##/": 298,
-      "##ç": 299,
-      "##ĥ": 300,
-      "##²": 301,
-      "##Ç": 302,
-      "##^": 303,
-      "##ć": 304,
-      "##j": 305,
-      "##v": 306,
-      "##~": 307,
-      "##N": 308,
-      "##Ń": 309,
-      "##q": 310,
-      "##ġ": 311,
-      "##È": 312,
-      "##ğ": 313,
-      "##ĝ": 314,
-      "##ø": 315,
-      "##æ": 316,
-      "##D": 317,
-      "##)": 318,
-      "##ģ": 319,
-      "##Ý": 320,
-      "##5": 321,
-      "##ĳ": 322,
-      "##3": 323,
-      "##ė": 324,
-      "##ķ": 325,
-      "##ò": 326,
-      "##®": 327,
-      "##x": 328,
-      "##t": 329,
-      "##;": 330,
-      "##¹": 331,
-      "##×": 332,
-      "##Û": 333,
-      "##k": 334,
-      "##ã": 335,
-      "##ċ": 336,
-      "##¤": 337,
-      "##&": 338,
-      "##Ä": 339,
-      "##b": 340,
-      "##Ñ": 341,
-      "##Ã": 342,
-      "##ú": 343,
-      "##F": 344,
-      "##¦": 345,
-      "##p": 346,
-      "##¼": 347,
-      "##ä": 348,
-      "##ï": 349,
-      "##¢": 350,
-      "##Ġ": 351,
-      "##î": 352,
-      "##Ĩ": 353,
-      "##Ü": 354,
-      "##·": 355,
-      "##¡": 356,
-      "##°": 357,
-      "##§": 358,
-      "##â": 359,
-      "##[": 360,
-      "##ď": 361,
-      "##ā": 362,
-      "##Þ": 363,
-      "##Ă": 364,
-      "##à": 365,
-      "##Ĥ": 366,
-      "##c": 367,
-      "##á": 368,
-      "##ê": 369,
-      "##ħ": 370,
-      "##ĕ": 371,
-      "##ñ": 372,
-      "##¯": 373,
-      "##Ĺ": 374,
-      "##Ö": 375,
-      "##Ć": 376,
-      "##T": 377,
-      "##Ĵ": 378,
-      "##>": 379,
-      "##+": 380,
-      "##ē": 381,
-      "##n": 382,
-      "##¥": 383,
-      "##Æ": 384,
-      "##X": 385,
-      "##³": 386,
-      "##'": 387,
-      "##P": 388,
-      "##Ě": 389,
-      "##ą": 390,
-      "##č": 391,
-      "##µ": 392,
-      "##R": 393,
-      "##Ï": 394,
-      "##ĵ": 395,
-      "##¸": 396,
-      "##Ľ": 397,
-      "##Ĝ": 398,
-      "##Õ": 399,
-      "##Ļ": 400,
-      "##J": 401,
-      "##»": 402,
-      "##ª": 403,
-      "##é": 404,
-      "##Ę": 405,
-      "##«": 406,
-      "##Ī": 407,
-      "##Ģ": 408,
-      "##?": 409,
-      "##Ķ": 410,
-      "##ł": 411,
-      "##ě": 412,
-      "##@": 413,
-      "##Ĕ": 414,
-      "##ī": 415,
-      "##Ĭ": 416,
-      "##`": 417,
-      "##Z": 418,
-      "##ý": 419,
-      "##]": 420,
-      "##(": 421,
-      "##Ē": 422,
-      "##}": 423,
-      "##<": 424,
-      "##K": 425,
-      "##e": 426,
-      "##É": 427,
-      "##.": 428,
-      "##å": 429,
-      "##A": 430,
-      "##f": 431,
-      "##M": 432,
-      "##1": 433,
-      "##G": 434,
-      "##Î": 435,
-      "##8": 436,
-      "##ù": 437,
-      "##d": 438,
-      "##z": 439,
-      "##£": 440,
-      "##Ā": 441,
-      "##ę": 442,
-      "##į": 443,
-      "##İ": 444,
-      "##ı": 445,
-      "##C": 446,
-      "##÷": 447,
-      "##ľ": 448,
-      "##ÿ": 449,
-      "##!": 450,
-      "##l": 451,
-      "##¶": 452,
-      "##Ą": 453,
-      "##Ċ": 454,
-      "##ó": 455,
-      "##ă": 456,
-      "##Í": 457,
-      "##0": 458,
-      "##À": 459,
-      "##2": 460,
-      "##¾": 461,
-      "##I": 462,
-      "##Ł": 463,
-      "##þ": 464,
-      "##O": 465,
-      "##Ò": 466,
-      "##B": 467,
-      "##\\": 468,
-      "##Ħ": 469,
-      "##H": 470,
-      "##õ": 471,
-      "##s": 472,
-      "##Ø": 473,
-      "##Ú": 474,
-      "##Á": 475,
-      "##Ô": 476,
-      "##í": 477,
-      "##L": 478,
       "##V": 479,
-      "##\"": 480,
-      "##û": 481,
-      "##ü": 482,
-      "##o": 483,
-      "##ĩ": 484,
-      "##a": 485,
-      "##ì": 486,
-      "##¬": 487,
-      "##S": 488,
-      "##Y": 489,
-      "##*": 490,
-      "##º": 491,
-      "##Ŀ": 492,
-      "##ĸ": 493,
-      "##h": 494,
-      "##¨": 495,
-      "##Į": 496,
-      "##-": 497,
-      "##±": 498,
-      "##6": 499,
-      "##%": 500,
-      "##|": 501,
-      "##Č": 502,
-      "##è": 503,
-      "##Ĉ": 504,
-      "##ô": 505,
       "##Ĳ": 506,
-      "##ŀ": 507,
-      "##đ": 508,
-      "##ß": 509,
-      "##U": 510,
-      "##,": 511,
-      "##Q": 512,
-      "##Ğ": 513,
       "<|unk|>": 514,
       "##in": 515,
       "##�": 516,

       "Ł": 255,
       "ł": 256,
       "Ń": 257,
+      "##S": 258,
+      "##E": 259,
+      "##_": 260,
+      "##Ğ": 261,
+      "##Ç": 262,
+      "##3": 263,
+      "##©": 264,
+      "##å": 265,
+      "##ê": 266,
+      "##Ć": 267,
+      "##{": 268,
+      "##!": 269,
+      "##ï": 270,
+      "##/": 271,
+      "##ĭ": 272,
+      "##ù": 273,
+      "##à": 274,
+      "##Ĉ": 275,
+      "##¤": 276,
+      "##*": 277,
+      "##ĩ": 278,
+      "##o": 279,
+      "##\\": 280,
+      "##č": 281,
+      "##6": 282,
+      "##¢": 283,
+      "##;": 284,
+      "##¶": 285,
+      "##B": 286,
+      "##û": 287,
+      "##@": 288,
+      "##z": 289,
+      "##ó": 290,
+      "##Ĵ": 291,
+      "##Ă": 292,
+      "##v": 293,
+      "##³": 294,
+      "##Ë": 295,
+      "##ę": 296,
+      "##÷": 297,
+      "##Č": 298,
+      "##L": 299,
+      "##½": 300,
+      "##Í": 301,
+      "##Ĩ": 302,
+      "##,": 303,
+      "##P": 304,
+      "##|": 305,
+      "##Ċ": 306,
+      "##k": 307,
+      "##'": 308,
+      "##ċ": 309,
+      "##į": 310,
+      "##>": 311,
+      "##I": 312,
+      "##2": 313,
+      "##9": 314,
+      "##K": 315,
+      "##Ô": 316,
+      "##ô": 317,
+      "##Ě": 318,
+      "##ğ": 319,
+      "##]": 320,
+      "##ø": 321,
+      "##C": 322,
+      "##A": 323,
+      "##a": 324,
+      "##H": 325,
+      "##«": 326,
+      "##×": 327,
+      "##ı": 328,
+      "##e": 329,
+      "##j": 330,
+      "##Á": 331,
+      "##i": 332,
+      "##p": 333,
+      "##Z": 334,
+      "##§": 335,
+      "##¯": 336,
+      "##~": 337,
+      "##ġ": 338,
+      "##À": 339,
+      "##ë": 340,
+      "##Î": 341,
+      "##þ": 342,
+      "##ď": 343,
+      "##ÿ": 344,
+      "##t": 345,
+      "##î": 346,
+      "##Ø": 347,
+      "##Â": 348,
+      "##ė": 349,
+      "##ò": 350,
+      "##ú": 351,
+      "##¡": 352,
+      "##µ": 353,
+      "##(": 354,
+      "##â": 355,
+      "##F": 356,
+      "##ĸ": 357,
+      "##ģ": 358,
+      "##ß": 359,
+      "##%": 360,
+      "##ī": 361,
+      "##É": 362,
+      "##đ": 363,
+      "##¥": 364,
+      "##O": 365,
+      "##ª": 366,
+      "##ā": 367,
+      "##Æ": 368,
+      "##^": 369,
+      "##£": 370,
+      "##ă": 371,
+      "##¬": 372,
+      "##q": 373,
+      "##-": 374,
+      "##Ė": 375,
+      "##ĳ": 376,
+      "##R": 377,
+      "##¼": 378,
+      "##Ĭ": 379,
+      "##ö": 380,
+      "##d": 381,
+      "##Ħ": 382,
+      "##c": 383,
+      "##+": 384,
+      "##Ö": 385,
+      "##ł": 386,
+      "##ŀ": 387,
+      "##Ñ": 388,
+      "##ē": 389,
+      "##È": 390,
+      "##¾": 391,
+      "##Ù": 392,
+      "##ĕ": 393,
+      "##ç": 394,
+      "##Ü": 395,
+      "##\"": 396,
+      "##f": 397,
+      "##º": 398,
+      "##ě": 399,
+      "##n": 400,
+      "##¨": 401,
+      "##Ã": 402,
+      "##Û": 403,
+      "##D": 404,
+      "##ĥ": 405,
+      "##1": 406,
+      "##ã": 407,
+      "##&": 408,
+      "##0": 409,
+      "##7": 410,
+      "##u": 411,
+      "##y": 412,
+      "##°": 413,
+      "##»": 414,
+      "##Ā": 415,
+      "##Ķ": 416,
+      "##Q": 417,
+      "##Ń": 418,
+      "##Ē": 419,
+      "##<": 420,
+      "##´": 421,
+      "##b": 422,
+      "##ĉ": 423,
+      "##¸": 424,
+      "##m": 425,
+      "##l": 426,
+      "##r": 427,
+      "##w": 428,
+      "##Ä": 429,
+      "##x": 430,
+      "##Ì": 431,
+      "##Ŀ": 432,
+      "##`": 433,
+      "##á": 434,
+      "##ą": 435,
+      "##Ę": 436,
+      "##Ð": 437,
+      "##Ĥ": 438,
+      "##)": 439,
+      "##²": 440,
+      "##¹": 441,
+      "##G": 442,
+      "##ħ": 443,
+      "##ĝ": 444,
+      "##õ": 445,
+      "##X": 446,
+      "##ć": 447,
+      "##Ď": 448,
+      "##·": 449,
+      "##İ": 450,
+      "##Þ": 451,
+      "##ķ": 452,
+      "##Į": 453,
+      "##W": 454,
+      "##Y": 455,
+      "##Ï": 456,
+      "##?": 457,
+      "##é": 458,
+      "##Ľ": 459,
+      "##Ó": 460,
+      "##[": 461,
+      "##Ą": 462,
+      "##ĺ": 463,
+      "##Ĕ": 464,
+      "##8": 465,
+      "##¦": 466,
+      "##Ú": 467,
+      "##h": 468,
+      "##U": 469,
+      "##ð": 470,
+      "##ñ": 471,
+      "##g": 472,
+      "##Ê": 473,
+      "##Ļ": 474,
+      "##è": 475,
+      "##5": 476,
+      "##:": 477,
+      "##=": 478,
       "##V": 479,
+      "##}": 480,
+      "##Å": 481,
+      "##Đ": 482,
+      "##Ī": 483,
+      "##$": 484,
+      "##ļ": 485,
+      "##Ġ": 486,
+      "##Ĺ": 487,
+      "##ý": 488,
+      "##4": 489,
+      "##Ģ": 490,
+      "##J": 491,
+      "##Ł": 492,
+      "##¿": 493,
+      "##M": 494,
+      "##s": 495,
+      "##Ý": 496,
+      "##®": 497,
+      "##ì": 498,
+      "##T": 499,
+      "##ü": 500,
+      "##Õ": 501,
+      "###": 502,
+      "##.": 503,
+      "##ä": 504,
+      "##Ĝ": 505,
       "##Ĳ": 506,
+      "##Ò": 507,
+      "##ĵ": 508,
+      "##ľ": 509,
+      "##±": 510,
+      "##N": 511,
+      "##í": 512,
+      "##æ": 513,
       "<|unk|>": 514,
       "##in": 515,
       "##�": 516,

fw57M_Entropy_thresholdB_16000/vocab.json CHANGED Viewed

The diff for this file is too large to render. See raw diff