| ".encode("utf-8"))]) + num_special_tokens | |
| loss = model(input_ids, labels=labels).loss | |
| loss.item() | |
| 2.66 | |
| For batched inference and training it is however recommended to make use of the tokenizer: | |
| thon | |
| from transformers import T5ForConditionalGeneration, AutoTokenizer | |
| model = T5ForConditionalGeneration.from_pretrained("google/byt5-small") | |
| tokenizer = AutoTokenizer.from_pretrained("google/byt5-small") | |
| model_inputs = tokenizer( | |
| ["Life is like a box of chocolates. |