cortex45mptformat / tokenizer /tokenizer_meta.json

Upload 7 files

c5ba719 verified 3 months ago

270 Bytes

{
  "vocab_size": 16000,
  "special_tokens": {
    "<|bos|>": 0,
    "<|eos|>": 1,
    "<|pad|>": 2,
    "<|sep|>": 3,
    "<|unk|>": 4,
    "<|sys|>": 5,
    "<|usr|>": 6,
    "<|ast|>": 7
  },
  "bos_id": 0,
  "eos_id": 1,
  "pad_id": 2,
  "sep_id": 3,
  "unk_id": 4
}

	{
	"vocab_size": 16000,
	"special_tokens": {
	"<\|bos\|>": 0,
	"<\|eos\|>": 1,
	"<\|pad\|>": 2,
	"<\|sep\|>": 3,
	"<\|unk\|>": 4,
	"<\|sys\|>": 5,
	"<\|usr\|>": 6,
	"<\|ast\|>": 7
	},
	"bos_id": 0,
	"eos_id": 1,
	"pad_id": 2,
	"sep_id": 3,
	"unk_id": 4
	}