bocss-lexer
v1.0.3
Published
Lexer to tokenize CSS
Readme
bocss-lexer
This module is a part of bocss.
You can found more documentation and follow advancement here.
More specifically here.
Description
Tokenize CSS content.
Installation
npm install —save bocss-lexer
Usage
Command
npx bocss-lexer [OPTIONS] content
- From file
npx bocss-lexer "$(< filename.css)"
- Into file
npx bocss-lexer "body { background-color: #FFF; color: #000; }" > filename.json
- From file into file
npx bocss-lexer "$(< filename.css)" > filename.json
Option
—help, -h: Print help message
Module
import tokenize from "bocss-lexer";
const content = `
:root {
--background: #FFF;
--foreground: #000;
}
body {
background-color: var(--background);
color: var(--foreground);
}
`;
console.log(tokenize(content));[
{
"type": "whitespace",
"value": "\n ",
"start": {
"cursor": 0,
"column": 0,
"line": 0
},
"next": {
"cursor": 3,
"column": 2,
"line": 1
}
},
{
"type": "punctuation",
"value": ":",
"start": {
"cursor": 3,
"column": 2,
"line": 1
},
"next": {
"cursor": 4,
"column": 3,
"line": 1
}
},
{
"type": "identifier",
"value": "root",
"start": {
"cursor": 4,
"column": 3,
"line": 1
},
"next": {
"cursor": 8,
"column": 7,
"line": 1
}
},
{
"type": "whitespace",
"value": " ",
"start": {
"cursor": 8,
"column": 7,
"line": 1
},
"next": {
"cursor": 9,
"column": 8,
"line": 1
}
},
{
"type": "punctuation",
"value": "{",
"start": {
"cursor": 9,
"column": 8,
"line": 1
},
"next": {
"cursor": 10,
"column": 9,
"line": 1
}
},
{
"type": "whitespace",
"value": "\n ",
"start": {
"cursor": 10,
"column": 9,
"line": 1
},
"next": {
"cursor": 15,
"column": 4,
"line": 2
}
},
{
"type": "operator",
"value": "-",
"start": {
"cursor": 15,
"column": 4,
"line": 2
},
"next": {
"cursor": 16,
"column": 5,
"line": 2
}
},
{
"type": "operator",
"value": "-",
"start": {
"cursor": 16,
"column": 5,
"line": 2
},
"next": {
"cursor": 17,
"column": 6,
"line": 2
}
},
{
"type": "identifier",
"value": "background",
"start": {
"cursor": 17,
"column": 6,
"line": 2
},
"next": {
"cursor": 27,
"column": 16,
"line": 2
}
},
{
"type": "punctuation",
"value": ":",
"start": {
"cursor": 27,
"column": 16,
"line": 2
},
"next": {
"cursor": 28,
"column": 17,
"line": 2
}
},
{
"type": "whitespace",
"value": " ",
"start": {
"cursor": 28,
"column": 17,
"line": 2
},
"next": {
"cursor": 29,
"column": 18,
"line": 2
}
},
{
"type": "color_hex",
"value": "#FFF",
"start": {
"cursor": 29,
"column": 18,
"line": 2
},
"next": {
"cursor": 33,
"column": 22,
"line": 2
}
},
{
"type": "punctuation",
"value": ";",
"start": {
"cursor": 33,
"column": 22,
"line": 2
},
"next": {
"cursor": 34,
"column": 23,
"line": 2
}
},
{
"type": "whitespace",
"value": "\n ",
"start": {
"cursor": 34,
"column": 23,
"line": 2
},
"next": {
"cursor": 39,
"column": 4,
"line": 3
}
},
{
"type": "operator",
"value": "-",
"start": {
"cursor": 39,
"column": 4,
"line": 3
},
"next": {
"cursor": 40,
"column": 5,
"line": 3
}
},
{
"type": "operator",
"value": "-",
"start": {
"cursor": 40,
"column": 5,
"line": 3
},
"next": {
"cursor": 41,
"column": 6,
"line": 3
}
},
{
"type": "identifier",
"value": "foreground",
"start": {
"cursor": 41,
"column": 6,
"line": 3
},
"next": {
"cursor": 51,
"column": 16,
"line": 3
}
},
{
"type": "punctuation",
"value": ":",
"start": {
"cursor": 51,
"column": 16,
"line": 3
},
"next": {
"cursor": 52,
"column": 17,
"line": 3
}
},
{
"type": "whitespace",
"value": " ",
"start": {
"cursor": 52,
"column": 17,
"line": 3
},
"next": {
"cursor": 53,
"column": 18,
"line": 3
}
},
{
"type": "color_hex",
"value": "#000",
"start": {
"cursor": 53,
"column": 18,
"line": 3
},
"next": {
"cursor": 57,
"column": 22,
"line": 3
}
},
{
"type": "punctuation",
"value": ";",
"start": {
"cursor": 57,
"column": 22,
"line": 3
},
"next": {
"cursor": 58,
"column": 23,
"line": 3
}
},
{
"type": "whitespace",
"value": "\n ",
"start": {
"cursor": 58,
"column": 23,
"line": 3
},
"next": {
"cursor": 61,
"column": 2,
"line": 4
}
},
{
"type": "punctuation",
"value": "}",
"start": {
"cursor": 61,
"column": 2,
"line": 4
},
"next": {
"cursor": 62,
"column": 3,
"line": 4
}
},
{
"type": "whitespace",
"value": "\n ",
"start": {
"cursor": 62,
"column": 3,
"line": 4
},
"next": {
"cursor": 65,
"column": 2,
"line": 5
}
},
{
"type": "identifier",
"value": "body",
"start": {
"cursor": 65,
"column": 2,
"line": 5
},
"next": {
"cursor": 69,
"column": 6,
"line": 5
}
},
{
"type": "whitespace",
"value": " ",
"start": {
"cursor": 69,
"column": 6,
"line": 5
},
"next": {
"cursor": 70,
"column": 7,
"line": 5
}
},
{
"type": "punctuation",
"value": "{",
"start": {
"cursor": 70,
"column": 7,
"line": 5
},
"next": {
"cursor": 71,
"column": 8,
"line": 5
}
},
{
"type": "whitespace",
"value": "\n ",
"start": {
"cursor": 71,
"column": 8,
"line": 5
},
"next": {
"cursor": 76,
"column": 4,
"line": 6
}
},
{
"type": "identifier",
"value": "background-color",
"start": {
"cursor": 76,
"column": 4,
"line": 6
},
"next": {
"cursor": 92,
"column": 20,
"line": 6
}
},
{
"type": "punctuation",
"value": ":",
"start": {
"cursor": 92,
"column": 20,
"line": 6
},
"next": {
"cursor": 93,
"column": 21,
"line": 6
}
},
{
"type": "whitespace",
"value": " ",
"start": {
"cursor": 93,
"column": 21,
"line": 6
},
"next": {
"cursor": 94,
"column": 22,
"line": 6
}
},
{
"type": "identifier",
"value": "var",
"start": {
"cursor": 94,
"column": 22,
"line": 6
},
"next": {
"cursor": 97,
"column": 25,
"line": 6
}
},
{
"type": "punctuation",
"value": "(",
"start": {
"cursor": 97,
"column": 25,
"line": 6
},
"next": {
"cursor": 98,
"column": 26,
"line": 6
}
},
{
"type": "operator",
"value": "-",
"start": {
"cursor": 98,
"column": 26,
"line": 6
},
"next": {
"cursor": 99,
"column": 27,
"line": 6
}
},
{
"type": "operator",
"value": "-",
"start": {
"cursor": 99,
"column": 27,
"line": 6
},
"next": {
"cursor": 100,
"column": 28,
"line": 6
}
},
{
"type": "identifier",
"value": "background",
"start": {
"cursor": 100,
"column": 28,
"line": 6
},
"next": {
"cursor": 110,
"column": 38,
"line": 6
}
},
{
"type": "punctuation",
"value": ")",
"start": {
"cursor": 110,
"column": 38,
"line": 6
},
"next": {
"cursor": 111,
"column": 39,
"line": 6
}
},
{
"type": "punctuation",
"value": ";",
"start": {
"cursor": 111,
"column": 39,
"line": 6
},
"next": {
"cursor": 112,
"column": 40,
"line": 6
}
},
{
"type": "whitespace",
"value": "\n ",
"start": {
"cursor": 112,
"column": 40,
"line": 6
},
"next": {
"cursor": 117,
"column": 4,
"line": 7
}
},
{
"type": "identifier",
"value": "color",
"start": {
"cursor": 117,
"column": 4,
"line": 7
},
"next": {
"cursor": 122,
"column": 9,
"line": 7
}
},
{
"type": "punctuation",
"value": ":",
"start": {
"cursor": 122,
"column": 9,
"line": 7
},
"next": {
"cursor": 123,
"column": 10,
"line": 7
}
},
{
"type": "whitespace",
"value": " ",
"start": {
"cursor": 123,
"column": 10,
"line": 7
},
"next": {
"cursor": 124,
"column": 11,
"line": 7
}
},
{
"type": "identifier",
"value": "var",
"start": {
"cursor": 124,
"column": 11,
"line": 7
},
"next": {
"cursor": 127,
"column": 14,
"line": 7
}
},
{
"type": "punctuation",
"value": "(",
"start": {
"cursor": 127,
"column": 14,
"line": 7
},
"next": {
"cursor": 128,
"column": 15,
"line": 7
}
},
{
"type": "operator",
"value": "-",
"start": {
"cursor": 128,
"column": 15,
"line": 7
},
"next": {
"cursor": 129,
"column": 16,
"line": 7
}
},
{
"type": "operator",
"value": "-",
"start": {
"cursor": 129,
"column": 16,
"line": 7
},
"next": {
"cursor": 130,
"column": 17,
"line": 7
}
},
{
"type": "identifier",
"value": "foreground",
"start": {
"cursor": 130,
"column": 17,
"line": 7
},
"next": {
"cursor": 140,
"column": 27,
"line": 7
}
},
{
"type": "punctuation",
"value": ")",
"start": {
"cursor": 140,
"column": 27,
"line": 7
},
"next": {
"cursor": 141,
"column": 28,
"line": 7
}
},
{
"type": "punctuation",
"value": ";",
"start": {
"cursor": 141,
"column": 28,
"line": 7
},
"next": {
"cursor": 142,
"column": 29,
"line": 7
}
},
{
"type": "whitespace",
"value": "\n ",
"start": {
"cursor": 142,
"column": 29,
"line": 7
},
"next": {
"cursor": 145,
"column": 2,
"line": 8
}
},
{
"type": "punctuation",
"value": "}",
"start": {
"cursor": 145,
"column": 2,
"line": 8
},
"next": {
"cursor": 146,
"column": 3,
"line": 8
}
}
]Types
type Position = { cursor: number, line: number, column: number };
type Element = { type: string, value: any, start?: Position, next?: Position };
type Token = Element & { value: string }
type AtKeyword = Token & { type: 'at_keyword' }; // @ followed by an identifier
type ColorHex = Token & { type: 'color_hex' }; // #000 or #000000
type CommentSingleLine = Token & { type: 'comment_single_line' }; // All between // and \n
type CommentMultipleLine = Token & { type: 'comment_multiple_line' }; // All between /* and */
type Identifier = Token & { type: 'identifier' }; /[\w\d_][\w\d_-]*/
type Number = Token & { type: 'number' }; // +12 or -.12 or +12.34e-56
type Operator = Token & { type: 'operator' }; /[+*/%=&|!><^~-]/
type Punctuation = Token & { type: 'punctuation' }; /[,;(){}[]:.#]/
type String = Token & { type: 'string' }; // single or double
type Whitespace = Token & { type: 'whitespace' }; /[\t\r\n ]/