mirror of
https://github.com/tenrok/BBob.git
synced 2026-05-21 13:24:05 +03:00
772d422d77
* feat(parser): first iteration of new lexer * feat(parser): convert token string props to number props * refactor(parser): optimize char grabber * refactor(parser): working on new lexer * refactor(parser): convert token string props to number props * refactor(parser): rebuild lexer, add tag attrs parsing * refactor(parser): rework word parsing and tag parsing * refactor(parser): rework to pass tests * refactor(parser): rework tag parsing * refactor(parser): rework escape tags parsing * refactor(parser): rework tests * refactor(parser): all test pass * refactor(parser): make lexer faster by move mode switching in loop * refactor(parser): remove all state map objects * refactor(parser): order of parsing states * refactor(parser): state switching without return * refactor(parser): rename buffers to chars * refactor(lexer): reduce function calls * feat(lexer): add new parser tests and code to pass it * fix(utils): remove unused variable in char grabber * feat(lexer): add test for new lexer bug * chore(*): add lexer and lexer2 to benchmark * chore(lexer): add some debug info for char grabber * feat(parser): add new test for single attributes without values * fix(lexer): paired tags tests * refactor(lexer): comment breaking changes tests for future releases * feat(core): improve tests * refactor(parser): add more tests, reduce char grabber size * refactor(parser): reduce utils size * refactor(parser): remove unused code from tag parsing code * refactor(parser): remove unused code from word to tag transforming code * chore(benchmark): fix benchmark imports
176 lines
3.7 KiB
JavaScript
176 lines
3.7 KiB
JavaScript
import {
|
|
OPEN_BRAKET,
|
|
CLOSE_BRAKET,
|
|
SLASH,
|
|
} from '@bbob/plugin-helper/lib/char';
|
|
|
|
// type, value, line, row,
|
|
const TOKEN_TYPE_ID = 'type'; // 0;
|
|
const TOKEN_VALUE_ID = 'value'; // 1;
|
|
const TOKEN_COLUMN_ID = 'row'; // 2;
|
|
const TOKEN_LINE_ID = 'line'; // 3;
|
|
|
|
const TOKEN_TYPE_WORD = 1; // 'word';
|
|
const TOKEN_TYPE_TAG = 2; // 'tag';
|
|
const TOKEN_TYPE_ATTR_NAME = 3; // 'attr-name';
|
|
const TOKEN_TYPE_ATTR_VALUE = 4; // 'attr-value';
|
|
const TOKEN_TYPE_SPACE = 5; // 'space';
|
|
const TOKEN_TYPE_NEW_LINE = 6; // 'new-line';
|
|
|
|
/**
|
|
* @param {Token} token
|
|
* @returns {string}
|
|
*/
|
|
const getTokenValue = (token) => {
|
|
if (token && typeof token[TOKEN_VALUE_ID] !== 'undefined') {
|
|
return token[TOKEN_VALUE_ID];
|
|
}
|
|
|
|
return '';
|
|
};
|
|
/**
|
|
* @param {Token}token
|
|
* @returns {number}
|
|
*/
|
|
const getTokenLine = (token) => (token && token[TOKEN_LINE_ID]) || 0;
|
|
const getTokenColumn = (token) => (token && token[TOKEN_COLUMN_ID]) || 0;
|
|
|
|
/**
|
|
* @param {Token} token
|
|
* @returns {boolean}
|
|
*/
|
|
const isTextToken = (token) => {
|
|
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
|
|
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_SPACE
|
|
|| token[TOKEN_TYPE_ID] === TOKEN_TYPE_NEW_LINE
|
|
|| token[TOKEN_TYPE_ID] === TOKEN_TYPE_WORD;
|
|
}
|
|
|
|
return false;
|
|
};
|
|
|
|
/**
|
|
* @param {Token} token
|
|
* @returns {boolean}
|
|
*/
|
|
const isTagToken = (token) => {
|
|
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
|
|
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_TAG;
|
|
}
|
|
|
|
return false;
|
|
};
|
|
const isTagEnd = (token) => getTokenValue(token).charCodeAt(0) === SLASH.charCodeAt(0);
|
|
const isTagStart = (token) => !isTagEnd(token);
|
|
const isAttrNameToken = (token) => {
|
|
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
|
|
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_ATTR_NAME;
|
|
}
|
|
|
|
return false;
|
|
};
|
|
|
|
/**
|
|
* @param {Token} token
|
|
* @returns {boolean}
|
|
*/
|
|
const isAttrValueToken = (token) => {
|
|
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
|
|
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_ATTR_VALUE;
|
|
}
|
|
|
|
return false;
|
|
};
|
|
|
|
const getTagName = (token) => {
|
|
const value = getTokenValue(token);
|
|
|
|
return isTagEnd(token) ? value.slice(1) : value;
|
|
};
|
|
|
|
const convertTagToText = (token) => {
|
|
let text = OPEN_BRAKET;
|
|
|
|
text += getTokenValue(token);
|
|
text += CLOSE_BRAKET;
|
|
|
|
return text;
|
|
};
|
|
|
|
class Token {
|
|
/**
|
|
* @param {String} type
|
|
* @param {String} value
|
|
* @param line
|
|
* @param row
|
|
*/
|
|
constructor(type, value, line, row) {
|
|
this[TOKEN_TYPE_ID] = Number(type);
|
|
this[TOKEN_VALUE_ID] = String(value);
|
|
this[TOKEN_LINE_ID] = Number(line);
|
|
this[TOKEN_COLUMN_ID] = Number(row);
|
|
}
|
|
|
|
isEmpty() {
|
|
// eslint-disable-next-line no-restricted-globals
|
|
return isNaN(this[TOKEN_TYPE_ID]);
|
|
}
|
|
|
|
isText() {
|
|
return isTextToken(this);
|
|
}
|
|
|
|
isTag() {
|
|
return isTagToken(this);
|
|
}
|
|
|
|
isAttrName() {
|
|
return isAttrNameToken(this);
|
|
}
|
|
|
|
isAttrValue() {
|
|
return isAttrValueToken(this);
|
|
}
|
|
|
|
isStart() {
|
|
return isTagStart(this);
|
|
}
|
|
|
|
isEnd() {
|
|
return isTagEnd(this);
|
|
}
|
|
|
|
getName() {
|
|
return getTagName(this);
|
|
}
|
|
|
|
getValue() {
|
|
return getTokenValue(this);
|
|
}
|
|
|
|
getLine() {
|
|
return getTokenLine(this);
|
|
}
|
|
|
|
getColumn() {
|
|
return getTokenColumn(this);
|
|
}
|
|
|
|
toString() {
|
|
return convertTagToText(this);
|
|
}
|
|
}
|
|
|
|
export const TYPE_ID = TOKEN_TYPE_ID;
|
|
export const VALUE_ID = TOKEN_VALUE_ID;
|
|
export const LINE_ID = TOKEN_LINE_ID;
|
|
export const COLUMN_ID = TOKEN_COLUMN_ID;
|
|
export const TYPE_WORD = TOKEN_TYPE_WORD;
|
|
export const TYPE_TAG = TOKEN_TYPE_TAG;
|
|
export const TYPE_ATTR_NAME = TOKEN_TYPE_ATTR_NAME;
|
|
export const TYPE_ATTR_VALUE = TOKEN_TYPE_ATTR_VALUE;
|
|
export const TYPE_SPACE = TOKEN_TYPE_SPACE;
|
|
export const TYPE_NEW_LINE = TOKEN_TYPE_NEW_LINE;
|
|
export { Token };
|
|
export default Token;
|