2
0
mirror of https://github.com/tenrok/BBob.git synced 2026-05-21 13:24:05 +03:00
Files
bbob/packages/bbob-parser/src/Token.js
T
Nikolay Kostyurin 772d422d77 feat(parser): rewrite lexer to make it faster (#50)
* feat(parser): first iteration of new lexer

* feat(parser): convert token string props to number props

* refactor(parser): optimize char grabber

* refactor(parser): working on new lexer

* refactor(parser): convert token string props to number props

* refactor(parser): rebuild lexer, add tag attrs parsing

* refactor(parser): rework word parsing and tag parsing

* refactor(parser): rework to pass tests

* refactor(parser): rework tag parsing

* refactor(parser): rework escape tags parsing

* refactor(parser): rework tests

* refactor(parser): all test pass

* refactor(parser): make lexer faster by move mode switching in loop

* refactor(parser): remove all state map objects

* refactor(parser): order of parsing states

* refactor(parser): state switching without return

* refactor(parser): rename buffers to chars

* refactor(lexer): reduce function calls

* feat(lexer): add new parser tests and code to pass it

* fix(utils): remove unused variable in char grabber

* feat(lexer): add test for new lexer bug

* chore(*): add lexer and lexer2 to benchmark

* chore(lexer): add some debug info for char grabber

* feat(parser): add new test for single attributes without values

* fix(lexer): paired tags tests

* refactor(lexer): comment breaking changes tests for future releases

* feat(core): improve tests

* refactor(parser): add more tests, reduce char grabber size

* refactor(parser): reduce utils size

* refactor(parser): remove unused code from tag parsing code

* refactor(parser): remove unused code from word to tag transforming code

* chore(benchmark): fix benchmark imports
2020-12-09 01:03:48 +02:00

176 lines
3.7 KiB
JavaScript

import {
OPEN_BRAKET,
CLOSE_BRAKET,
SLASH,
} from '@bbob/plugin-helper/lib/char';
// type, value, line, row,
const TOKEN_TYPE_ID = 'type'; // 0;
const TOKEN_VALUE_ID = 'value'; // 1;
const TOKEN_COLUMN_ID = 'row'; // 2;
const TOKEN_LINE_ID = 'line'; // 3;
const TOKEN_TYPE_WORD = 1; // 'word';
const TOKEN_TYPE_TAG = 2; // 'tag';
const TOKEN_TYPE_ATTR_NAME = 3; // 'attr-name';
const TOKEN_TYPE_ATTR_VALUE = 4; // 'attr-value';
const TOKEN_TYPE_SPACE = 5; // 'space';
const TOKEN_TYPE_NEW_LINE = 6; // 'new-line';
/**
* @param {Token} token
* @returns {string}
*/
const getTokenValue = (token) => {
if (token && typeof token[TOKEN_VALUE_ID] !== 'undefined') {
return token[TOKEN_VALUE_ID];
}
return '';
};
/**
* @param {Token}token
* @returns {number}
*/
const getTokenLine = (token) => (token && token[TOKEN_LINE_ID]) || 0;
const getTokenColumn = (token) => (token && token[TOKEN_COLUMN_ID]) || 0;
/**
* @param {Token} token
* @returns {boolean}
*/
const isTextToken = (token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_SPACE
|| token[TOKEN_TYPE_ID] === TOKEN_TYPE_NEW_LINE
|| token[TOKEN_TYPE_ID] === TOKEN_TYPE_WORD;
}
return false;
};
/**
* @param {Token} token
* @returns {boolean}
*/
const isTagToken = (token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_TAG;
}
return false;
};
const isTagEnd = (token) => getTokenValue(token).charCodeAt(0) === SLASH.charCodeAt(0);
const isTagStart = (token) => !isTagEnd(token);
const isAttrNameToken = (token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_ATTR_NAME;
}
return false;
};
/**
* @param {Token} token
* @returns {boolean}
*/
const isAttrValueToken = (token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_ATTR_VALUE;
}
return false;
};
const getTagName = (token) => {
const value = getTokenValue(token);
return isTagEnd(token) ? value.slice(1) : value;
};
const convertTagToText = (token) => {
let text = OPEN_BRAKET;
text += getTokenValue(token);
text += CLOSE_BRAKET;
return text;
};
class Token {
/**
* @param {String} type
* @param {String} value
* @param line
* @param row
*/
constructor(type, value, line, row) {
this[TOKEN_TYPE_ID] = Number(type);
this[TOKEN_VALUE_ID] = String(value);
this[TOKEN_LINE_ID] = Number(line);
this[TOKEN_COLUMN_ID] = Number(row);
}
isEmpty() {
// eslint-disable-next-line no-restricted-globals
return isNaN(this[TOKEN_TYPE_ID]);
}
isText() {
return isTextToken(this);
}
isTag() {
return isTagToken(this);
}
isAttrName() {
return isAttrNameToken(this);
}
isAttrValue() {
return isAttrValueToken(this);
}
isStart() {
return isTagStart(this);
}
isEnd() {
return isTagEnd(this);
}
getName() {
return getTagName(this);
}
getValue() {
return getTokenValue(this);
}
getLine() {
return getTokenLine(this);
}
getColumn() {
return getTokenColumn(this);
}
toString() {
return convertTagToText(this);
}
}
export const TYPE_ID = TOKEN_TYPE_ID;
export const VALUE_ID = TOKEN_VALUE_ID;
export const LINE_ID = TOKEN_LINE_ID;
export const COLUMN_ID = TOKEN_COLUMN_ID;
export const TYPE_WORD = TOKEN_TYPE_WORD;
export const TYPE_TAG = TOKEN_TYPE_TAG;
export const TYPE_ATTR_NAME = TOKEN_TYPE_ATTR_NAME;
export const TYPE_ATTR_VALUE = TOKEN_TYPE_ATTR_VALUE;
export const TYPE_SPACE = TOKEN_TYPE_SPACE;
export const TYPE_NEW_LINE = TOKEN_TYPE_NEW_LINE;
export { Token };
export default Token;