ByteSpanTokenisers
/

tokenizers

Model card Files Files and versions Community

Zeb commited on 23 days ago

Commit

583e43f

1 Parent(s): 8b5bd01

Rename all tokenizers

Browse files

This view is limited to 50 files because it contains too many changes. See raw diff

Files changed (50) hide show

.DS_Store +0 -0
.gitattributes +3 -0
{fw57M_Entropy_bytespanP0-0_128000 → BPEWP_128000}/special_tokens_map.json +0 -0
{fw57M_Entropy_bytespanP0-0_128000 → BPEWP_128000}/tokenizer.json +0 -0
{fw57M_Entropy_bytespanP0-0_128000 → BPEWP_128000}/tokenizer_config.json +0 -0
{fw57M_Entropy_bytespanP0-0_128000 → BPEWP_128000}/vocab.json +0 -0
{fw57M_Entropy_bytespanP0-0_16000 → BPEWP_16000}/special_tokens_map.json +0 -0
{fw57M_Entropy_bytespanP0-0_16000 → BPEWP_16000}/tokenizer.json +0 -0
{fw57M_Entropy_bytespanP0-0_16000 → BPEWP_16000}/tokenizer_config.json +0 -0
{fw57M_Entropy_bytespanP0-0_16000 → BPEWP_16000}/vocab.json +0 -0
{fw57M_Entropy_bytespanP0-0_32000 → BPEWP_32000}/special_tokens_map.json +0 -0
{fw57M_Entropy_bytespanP0-0_32000 → BPEWP_32000}/tokenizer.json +0 -0
{fw57M_Entropy_bytespanP0-0_32000 → BPEWP_32000}/tokenizer_config.json +0 -0
{fw57M_Entropy_bytespanP0-0_32000 → BPEWP_32000}/vocab.json +0 -0
{fw57M_Entropy_bytespanP0-0_64000 → BPEWP_64000}/special_tokens_map.json +0 -0
{fw57M_Entropy_bytespanP0-0_64000 → BPEWP_64000}/tokenizer.json +0 -0
{fw57M_Entropy_bytespanP0-0_64000 → BPEWP_64000}/tokenizer_config.json +0 -0
{fw57M_Entropy_bytespanP0-0_64000 → BPEWP_64000}/vocab.json +0 -0
{fw57M_Entropy_bytespanP0-0_8064 → BPEWP_8064}/special_tokens_map.json +0 -0
{fw57M_Entropy_bytespanP0-0_8064 → BPEWP_8064}/tokenizer.json +0 -0
{fw57M_Entropy_bytespanP0-0_8064 → BPEWP_8064}/tokenizer_config.json +0 -0
{fw57M_Entropy_bytespanP0-0_8064 → BPEWP_8064}/vocab.json +0 -0
{frequency_128000 → BPE_128000}/merges.txt +0 -0
{frequency_128000 → BPE_128000}/merges_data.csv +0 -0
{frequency_128000 → BPE_128000}/special_tokens_map.json +0 -0
{frequency_128000 → BPE_128000}/tokenizer.json +0 -0
{frequency_128000 → BPE_128000}/tokenizer_config.json +0 -0
{frequency_128000 → BPE_128000}/vocab.json +0 -0
{frequency_16000 → BPE_16000}/merges.txt +0 -0
{frequency_16000 → BPE_16000}/merges_data.csv +0 -0
{frequency_16000 → BPE_16000}/special_tokens_map.json +0 -0
{frequency_16000 → BPE_16000}/tokenizer.json +0 -0
{frequency_16000 → BPE_16000}/tokenizer_config.json +0 -0
{frequency_16000 → BPE_16000}/vocab.json +0 -0
{frequency_32000 → BPE_32000}/merges.txt +0 -0
{frequency_32000 → BPE_32000}/merges_data.csv +0 -0
{frequency_32000 → BPE_32000}/special_tokens_map.json +0 -0
{frequency_32000 → BPE_32000}/tokenizer.json +0 -0
{frequency_32000 → BPE_32000}/tokenizer_config.json +0 -0
{frequency_32000 → BPE_32000}/vocab.json +0 -0
{frequency_64000 → BPE_64000}/merges.txt +0 -0
{frequency_64000 → BPE_64000}/merges_data.csv +0 -0
{frequency_64000 → BPE_64000}/special_tokens_map.json +0 -0
{frequency_64000 → BPE_64000}/tokenizer.json +0 -0
{frequency_64000 → BPE_64000}/tokenizer_config.json +0 -0
{frequency_64000 → BPE_64000}/vocab.json +0 -0
{frequency_8064 → BPE_8064}/merges.txt +0 -0
{frequency_8064 → BPE_8064}/merges_data.csv +0 -0
{frequency_8064 → BPE_8064}/special_tokens_map.json +0 -0
{frequency_8064 → BPE_8064}/tokenizer.json +0 -0

.DS_Store ADDED Viewed

Binary file (6.15 kB). View file

.gitattributes CHANGED Viewed

@@ -64,3 +64,6 @@ frequencymultirussian_256000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 frequencymultirussian_256000/vocab.json filter=lfs diff=lfs merge=lfs -text
 fw57M_Surprisal_thresholdB_64000/stats.csv filter=lfs diff=lfs merge=lfs -text
 fw57M_Entropy_thresholdB_64000/stats.csv filter=lfs diff=lfs merge=lfs -text

 frequencymultirussian_256000/vocab.json filter=lfs diff=lfs merge=lfs -text
 fw57M_Surprisal_thresholdB_64000/stats.csv filter=lfs diff=lfs merge=lfs -text
 fw57M_Entropy_thresholdB_64000/stats.csv filter=lfs diff=lfs merge=lfs -text
+ByteSpanEntropyGlobalIncrement_64000/stats.csv filter=lfs diff=lfs merge=lfs -text
+ByteSpanSurprisalGlobalIncrement_64000/stats.csv filter=lfs diff=lfs merge=lfs -text
+MultiBPE_256000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

{fw57M_Entropy_bytespanP0-0_128000 → BPEWP_128000}/special_tokens_map.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_128000 → BPEWP_128000}/tokenizer.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_128000 → BPEWP_128000}/tokenizer_config.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_128000 → BPEWP_128000}/vocab.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_16000 → BPEWP_16000}/special_tokens_map.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_16000 → BPEWP_16000}/tokenizer.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_16000 → BPEWP_16000}/tokenizer_config.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_16000 → BPEWP_16000}/vocab.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_32000 → BPEWP_32000}/special_tokens_map.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_32000 → BPEWP_32000}/tokenizer.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_32000 → BPEWP_32000}/tokenizer_config.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_32000 → BPEWP_32000}/vocab.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_64000 → BPEWP_64000}/special_tokens_map.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_64000 → BPEWP_64000}/tokenizer.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_64000 → BPEWP_64000}/tokenizer_config.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_64000 → BPEWP_64000}/vocab.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_8064 → BPEWP_8064}/special_tokens_map.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_8064 → BPEWP_8064}/tokenizer.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_8064 → BPEWP_8064}/tokenizer_config.json RENAMED Viewed

File without changes

{fw57M_Entropy_bytespanP0-0_8064 → BPEWP_8064}/vocab.json RENAMED Viewed

File without changes

{frequency_128000 → BPE_128000}/merges.txt RENAMED Viewed

File without changes

{frequency_128000 → BPE_128000}/merges_data.csv RENAMED Viewed

File without changes

{frequency_128000 → BPE_128000}/special_tokens_map.json RENAMED Viewed

File without changes

{frequency_128000 → BPE_128000}/tokenizer.json RENAMED Viewed

File without changes

{frequency_128000 → BPE_128000}/tokenizer_config.json RENAMED Viewed

File without changes

{frequency_128000 → BPE_128000}/vocab.json RENAMED Viewed

File without changes

{frequency_16000 → BPE_16000}/merges.txt RENAMED Viewed

File without changes

{frequency_16000 → BPE_16000}/merges_data.csv RENAMED Viewed

File without changes

{frequency_16000 → BPE_16000}/special_tokens_map.json RENAMED Viewed

File without changes

{frequency_16000 → BPE_16000}/tokenizer.json RENAMED Viewed

File without changes

{frequency_16000 → BPE_16000}/tokenizer_config.json RENAMED Viewed

File without changes

{frequency_16000 → BPE_16000}/vocab.json RENAMED Viewed

File without changes

{frequency_32000 → BPE_32000}/merges.txt RENAMED Viewed

File without changes

{frequency_32000 → BPE_32000}/merges_data.csv RENAMED Viewed

File without changes

{frequency_32000 → BPE_32000}/special_tokens_map.json RENAMED Viewed

File without changes

{frequency_32000 → BPE_32000}/tokenizer.json RENAMED Viewed

File without changes

{frequency_32000 → BPE_32000}/tokenizer_config.json RENAMED Viewed

File without changes

{frequency_32000 → BPE_32000}/vocab.json RENAMED Viewed

File without changes

{frequency_64000 → BPE_64000}/merges.txt RENAMED Viewed

File without changes

{frequency_64000 → BPE_64000}/merges_data.csv RENAMED Viewed

File without changes

{frequency_64000 → BPE_64000}/special_tokens_map.json RENAMED Viewed

File without changes

{frequency_64000 → BPE_64000}/tokenizer.json RENAMED Viewed

File without changes

{frequency_64000 → BPE_64000}/tokenizer_config.json RENAMED Viewed

File without changes

{frequency_64000 → BPE_64000}/vocab.json RENAMED Viewed

File without changes

{frequency_8064 → BPE_8064}/merges.txt RENAMED Viewed

File without changes

{frequency_8064 → BPE_8064}/merges_data.csv RENAMED Viewed

File without changes

{frequency_8064 → BPE_8064}/special_tokens_map.json RENAMED Viewed

File without changes

{frequency_8064 → BPE_8064}/tokenizer.json RENAMED Viewed

File without changes