abstract-scanner
v2.0.2
Published
Universal scanner for lexical analysis.
Downloads
24
Readme
abstract-scanner
Universal scanner for lexical analysis.
The whole project are written with typescript
, but it's ok to use it as a common nodejs module.
Of course, you can find the umd
version of this module in lib/*
.
Install
npm install abstract-scanner --save
Exports
Token
: scan result wrapper.AbstractScanner
: abstract scanner needed to be extended.utils
: namespace of some helpful functions .version
: project version.
Interface
class Token {
type: string;
loc: SourceLocation;
source: string;
value?: any;
message?: string | string[];
constructor(type: string, source: string, start: Position, end: Position);
}
interface Position {
index: number;
line: number;
column: number;
}
interface SourceLocation {
readonly start: Position;
readonly end: Position;
}
interface Config {
line?: number;
column?: number;
start?: number;
end?: number;
}
abstract class AbstractScanner {
readonly source: string;
readonly length: number;
marker: Position;
private scanStartingMarker;
private scanEndingMarker;
constructor(source: string, config?: Config);
saveState(): Position;
restoreState(state: Position): void;
startScan(): void;
endScan(): void;
clear(): void;
getScanLength(): number;
getScanResult(): string;
constructToken(type: string): Token;
constructIllegalToken(message?: string | string[]): Token;
eof(): boolean;
peek(length?: number): string;
move(offset?: number): void;
moveWhen(judge: (cc: number, ch?: string) => boolean): void;
fromCodePoint(cp: number): string;
fromCharCode(cc: number): string;
getCodePoint(offset?: number): number;
getCharCode(offset?: number): number;
scanLineTerminator(): string;
scanBlankSpace(): string;
skipSpace(): void;
}
namespace utils {
function isWhiteSpace(cp: number): boolean;
function isLineTerminator(cp: number): boolean;
function isDecimalDigit(cp: number): boolean;
function isHexDigit(cp: number): boolean;
function isOctalDigit(cp: number): boolean;
function isBinaryDigit(cp: number): boolean;
namespace JS {
isIdentifierStart(cp: number): boolean;
isIdentifierPart(cp: number): boolean;
isKeyword(id: string): boolean;
isFutureReservedWord(id: string): boolean;
isStrictModeReservedWord(id: string): boolean;
};
}
Example
A full version scanner of tokenzing javascript is in directory examples/
.
For an illustrative purpose, here's an example of how to scan the NumericLiteral of javascript.
import { Token, AbstractScanner, utils } from 'abstract-scanner';
class Scanner extends AbstractScanner {
scanHexLiteral(): Token | null {
const str = this.peek(2);
if (str[0] === '0' && (str[1] === 'x' || str[1] === 'X')) {
this.startScan();
this.move(2);
this.moveWhen(cp => utils.isHexDigit(cp));
this.endScan();
if (this.getScanLength() === 2) {
return this.constructIllegalToken();
} else {
const token = this.constructToken('NumericLiteral');
token.value = parseInt(token.source, 16);
return token;
}
}
return null;
}
scanBinaryLiteral(): Token | null {...}
scanOctalLiteral(): Token | null {...}
scanDecimalLiteral(): Token | null {...}
nextToken(): Token | null {
return this.scanHexLiteral() || this.scanOctalLiteral()
|| this.scanBinaryLiteral() || this.scanDecimalLiteral();
}
}