InfoTokenizers
/

tokenizers

Model card Files Files and versions Community

Zeb commited on Apr 17

Commit

2e9392c

1 Parent(s): 78a72a8

Update frequency tokenizer names

Browse files

Files changed (37) hide show

.gitattributes +2 -0
{fw57M_Entropy_frequency_128000 → frequency_128000}/merges.txt +0 -0
{fw57M_Entropy_frequency_128000 → frequency_128000}/merges_data.csv +0 -0
{fw57M_Entropy_frequency_128000 → frequency_128000}/special_tokens_map.json +0 -0
{fw57M_Entropy_frequency_128000 → frequency_128000}/tokenizer.json +0 -0
{fw57M_Entropy_frequency_128000 → frequency_128000}/tokenizer_config.json +0 -0
{fw57M_Entropy_frequency_128000 → frequency_128000}/vocab.json +0 -0
{fw57M_Entropy_frequency_16000 → frequency_16000/fw57M_Entropy_frequency_16000}/merges.txt +0 -0
{fw57M_Entropy_frequency_16000 → frequency_16000/fw57M_Entropy_frequency_16000}/merges_data.csv +0 -0
{fw57M_Entropy_frequency_16000 → frequency_16000/fw57M_Entropy_frequency_16000}/special_tokens_map.json +0 -0
{fw57M_Entropy_frequency_16000 → frequency_16000/fw57M_Entropy_frequency_16000}/tokenizer.json +0 -0
{fw57M_Entropy_frequency_16000 → frequency_16000/fw57M_Entropy_frequency_16000}/tokenizer_config.json +0 -0
{fw57M_Entropy_frequency_16000 → frequency_16000/fw57M_Entropy_frequency_16000}/vocab.json +0 -0
{fw57M_Entropy_frequency_256000 → frequency_256000}/merges.txt +0 -0
{fw57M_Entropy_frequency_256000 → frequency_256000}/merges_data.csv +0 -0
{fw57M_Entropy_frequency_256000 → frequency_256000}/special_tokens_map.json +0 -0
{fw57M_Entropy_frequency_256000 → frequency_256000}/tokenizer.json +0 -0
{fw57M_Entropy_frequency_256000 → frequency_256000}/tokenizer_config.json +0 -0
{fw57M_Entropy_frequency_256000 → frequency_256000}/vocab.json +0 -0
{fw57M_Entropy_frequency_32000 → frequency_32000}/merges.txt +0 -0
{fw57M_Entropy_frequency_32000 → frequency_32000}/merges_data.csv +0 -0
{fw57M_Entropy_frequency_32000 → frequency_32000}/special_tokens_map.json +0 -0
{fw57M_Entropy_frequency_32000 → frequency_32000}/tokenizer.json +0 -0
{fw57M_Entropy_frequency_32000 → frequency_32000}/tokenizer_config.json +0 -0
{fw57M_Entropy_frequency_32000 → frequency_32000}/vocab.json +0 -0
{fw57M_Entropy_frequency_64000 → frequency_64000}/merges.txt +0 -0
{fw57M_Entropy_frequency_64000 → frequency_64000}/merges_data.csv +0 -0
{fw57M_Entropy_frequency_64000 → frequency_64000}/special_tokens_map.json +0 -0
{fw57M_Entropy_frequency_64000 → frequency_64000}/tokenizer.json +0 -0
{fw57M_Entropy_frequency_64000 → frequency_64000}/tokenizer_config.json +0 -0
{fw57M_Entropy_frequency_64000 → frequency_64000}/vocab.json +0 -0
{fw57M_Entropy_frequency_8064 → frequency_8064}/merges.txt +0 -0
{fw57M_Entropy_frequency_8064 → frequency_8064}/merges_data.csv +0 -0
{fw57M_Entropy_frequency_8064 → frequency_8064}/special_tokens_map.json +0 -0
{fw57M_Entropy_frequency_8064 → frequency_8064}/tokenizer.json +0 -0
{fw57M_Entropy_frequency_8064 → frequency_8064}/tokenizer_config.json +0 -0
{fw57M_Entropy_frequency_8064 → frequency_8064}/vocab.json +0 -0

.gitattributes CHANGED Viewed

@@ -35,3 +35,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 fw57M_Entropy_frequency_256000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 fw57M_Entropy_min-mean-post-merge_256000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

 *tfevents* filter=lfs diff=lfs merge=lfs -text
 fw57M_Entropy_frequency_256000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 fw57M_Entropy_min-mean-post-merge_256000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+frequency_256000/tokenizer filter=lfs diff=lfs merge=lfs -text
+frequency_256000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

{fw57M_Entropy_frequency_128000 → frequency_128000}/merges.txt RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_128000 → frequency_128000}/merges_data.csv RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_128000 → frequency_128000}/special_tokens_map.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_128000 → frequency_128000}/tokenizer.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_128000 → frequency_128000}/tokenizer_config.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_128000 → frequency_128000}/vocab.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_16000 → frequency_16000/fw57M_Entropy_frequency_16000}/merges.txt RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_16000 → frequency_16000/fw57M_Entropy_frequency_16000}/merges_data.csv RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_16000 → frequency_16000/fw57M_Entropy_frequency_16000}/special_tokens_map.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_16000 → frequency_16000/fw57M_Entropy_frequency_16000}/tokenizer.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_16000 → frequency_16000/fw57M_Entropy_frequency_16000}/tokenizer_config.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_16000 → frequency_16000/fw57M_Entropy_frequency_16000}/vocab.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_256000 → frequency_256000}/merges.txt RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_256000 → frequency_256000}/merges_data.csv RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_256000 → frequency_256000}/special_tokens_map.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_256000 → frequency_256000}/tokenizer.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_256000 → frequency_256000}/tokenizer_config.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_256000 → frequency_256000}/vocab.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_32000 → frequency_32000}/merges.txt RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_32000 → frequency_32000}/merges_data.csv RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_32000 → frequency_32000}/special_tokens_map.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_32000 → frequency_32000}/tokenizer.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_32000 → frequency_32000}/tokenizer_config.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_32000 → frequency_32000}/vocab.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_64000 → frequency_64000}/merges.txt RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_64000 → frequency_64000}/merges_data.csv RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_64000 → frequency_64000}/special_tokens_map.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_64000 → frequency_64000}/tokenizer.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_64000 → frequency_64000}/tokenizer_config.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_64000 → frequency_64000}/vocab.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_8064 → frequency_8064}/merges.txt RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_8064 → frequency_8064}/merges_data.csv RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_8064 → frequency_8064}/special_tokens_map.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_8064 → frequency_8064}/tokenizer.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_8064 → frequency_8064}/tokenizer_config.json RENAMED Viewed

File without changes

{fw57M_Entropy_frequency_8064 → frequency_8064}/vocab.json RENAMED Viewed

File without changes