Upload 3 files

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<bos>",
+  "eos_token": "<eos>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenizer.json

@@ -0,0 +1,405 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<bos>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "<eos>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "NFKC"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "A"
+        },
+        "content": "↨a"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "B"
+        },
+        "content": "↨b"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "C"
+        },
+        "content": "↨c"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "D"
+        },
+        "content": "↨d"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "E"
+        },
+        "content": "↨e"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "F"
+        },
+        "content": "↨f"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "G"
+        },
+        "content": "↨g"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "H"
+        },
+        "content": "↨h"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "I"
+        },
+        "content": "↨i"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "J"
+        },
+        "content": "↨j"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "K"
+        },
+        "content": "↨k"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "L"
+        },
+        "content": "↨l"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "M"
+        },
+        "content": "↨m"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "N"
+        },
+        "content": "↨n"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "O"
+        },
+        "content": "↨o"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "P"
+        },
+        "content": "↨p"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "Q"
+        },
+        "content": "↨q"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "R"
+        },
+        "content": "↨r"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "S"
+        },
+        "content": "↨s"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "T"
+        },
+        "content": "↨t"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "U"
+        },
+        "content": "↨u"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "V"
+        },
+        "content": "↨v"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "W"
+        },
+        "content": "↨w"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "X"
+        },
+        "content": "↨x"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "Y"
+        },
+        "content": "↨y"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "Z"
+        },
+        "content": "↨z"
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Split",
+    "pattern": {
+      "Regex": "\\X"
+    },
+    "behavior": "Isolated",
+    "invert": false
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 0
+        }
+      }
+    ],
+    "special_tokens": {}
+  },
+  "decoder": {
+    "type": "Sequence",
+    "decoders": []
+  },
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "<pad>": 0,
+      "<unk>": 1,
+      "<bos>": 2,
+      "<eos>": 3,
+      "↨": 4,
+      "\n": 5,
+      "\t": 6,
+      " ": 7,
+      "0": 8,
+      "1": 9,
+      "2": 10,
+      "3": 11,
+      "4": 12,
+      "5": 13,
+      "6": 14,
+      "7": 15,
+      "8": 16,
+      "9": 17,
+      "a": 18,
+      "b": 19,
+      "c": 20,
+      "d": 21,
+      "e": 22,
+      "f": 23,
+      "g": 24,
+      "h": 25,
+      "i": 26,
+      "j": 27,
+      "k": 28,
+      "l": 29,
+      "m": 30,
+      "n": 31,
+      "o": 32,
+      "p": 33,
+      "q": 34,
+      "r": 35,
+      "s": 36,
+      "t": 37,
+      "u": 38,
+      "v": 39,
+      "w": 40,
+      "x": 41,
+      "y": 42,
+      "z": 43,
+      "\"": 44,
+      "!": 45,
+      "$": 46,
+      "&": 47,
+      "'": 48,
+      "#": 49,
+      ",": 50,
+      "/": 51,
+      "+": 52,
+      "=": 53,
+      "-": 54,
+      "<": 55,
+      ">": 56,
+      "*": 57,
+      "@": 58,
+      ".": 59,
+      ":": 60,
+      ";": 61,
+      "[": 62,
+      "]": 63,
+      "{": 64,
+      "}": 65,
+      "(": 66,
+      ")": 67,
+      "^": 68,
+      "_": 69,
+      "?": 70,
+      "è": 71,
+      "é": 72,
+      "¤69": 73,
+      "¤70": 74,
+      "¤71": 75,
+      "¤72": 76,
+      "¤73": 77,
+      "¤74": 78,
+      "¤75": 79,
+      "¤76": 80,
+      "¤77": 81,
+      "¤78": 82,
+      "¤79": 83,
+      "¤80": 84,
+      "¤81": 85,
+      "¤82": 86,
+      "¤83": 87,
+      "¤84": 88,
+      "¤85": 89,
+      "¤86": 90,
+      "¤87": 91,
+      "¤88": 92,
+      "¤89": 93,
+      "¤90": 94,
+      "¤91": 95,
+      "¤92": 96,
+      "¤93": 97,
+      "¤94": 98,
+      "¤95": 99,
+      "¤96": 100,
+      "¤97": 101,
+      "¤98": 102,
+      "¤99": 103,
+      "¤100": 104,
+      "¤101": 105,
+      "¤102": 106,
+      "¤103": 107,
+      "¤104": 108,
+      "¤105": 109,
+      "¤106": 110,
+      "¤107": 111,
+      "¤108": 112,
+      "¤109": 113,
+      "¤110": 114,
+      "¤111": 115,
+      "¤112": 116,
+      "¤113": 117,
+      "¤114": 118,
+      "¤115": 119,
+      "¤116": 120,
+      "¤117": 121,
+      "¤118": 122,
+      "¤119": 123,
+      "¤120": 124,
+      "¤121": 125,
+      "¤122": 126,
+      "¤123": 127
+    },
+    "unk_token": "<unk>"
+  }
+}

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizer",
+  "unk_token": "<unk>"
+}