@kessler/tokenize
v2.0.0
Published
personal tokenize util
Downloads
13
Readme
tokenize
personal tokenize util
example
npm i -S @kessler/tokenize
const tokenize = require('@kessler/tokenize')
let tokens1 = tokenize('text')
// with non default tokenization rules
const rules = {
tokenChars: ['(', ')', ',', '[', ']'],
globbingChars: ['"', '\''],
separators: [' ']
}
let tokens2 = tokenize('(text) "asd" blabla', rules)license
MIT © Yaniv Kessler
