toxe
v1.1.3
Published
SentencePiece Tokenizer for CrossEncoders
Downloads
47
Readme
toxe
Install
npm i toxeUsage
import { Toxe } from 'toxe';
const toxe = new Toxe('./spm.model', {
bos: 1,
eos: 2
});
const ids = await toxe.encode("a", [
"a b",
"a b c",
]);Credits
https://github.com/JanKaul/sentencepiece
