tokenizer_config.json

{
  "add_prefix_space": false,
  "auto_map": {
    "AutoTokenizer": [
      null,
      "tokenization_codify_fast.CodifyTokenizerFast"
    ]
  },
  "bos_token": "<|endoftext|>",
  "eos_token": "<|endoftext|>",
  "name_or_path": "smallcloudai/codify_medium_multi",
  "special_tokens_map_file": "/home/user/.cache/huggingface/hub/models--smallcloudai--codify_medium_multi/snapshots/ae6835df6784621072944fcc3403ca89f1308f96/special_tokens_map.json",
  "tokenizer_class": "CodifyTokenizer",
  "unk_token": "<|endoftext|>"
}