mirrors/bbob
2
0
Fork 0
mirror of https://github.com/tenrok/BBob.git synced 2026-06-08 17:22:26 +03:00
Code Activity

feat: typescript support (#185)

Browse Source 
* feat: initial typescript support

* feat: typescript support

* feat(plugin-helper): move files to typescript

* chore: update lock files

* feat: preset types

* fix: build

* fix: benchmark

* fix: remove pnpm cache

* fix: bench action

* fix: pnpm recursive install

* fix: nx cache

* fix: lock file

* fix: workflows

* fix: lerna support in pnpm

* fix: pnpm workspace

* fix: remove unused files

* fix: pnpm lock file

* fix: update lerna for support pnpm

* fix: lerna bootstrap

* fix: rollup build

* fix: update nx

* fix: build

* fix: add nx dep target

* fix: remove nx cache

* fix: workflow run on push only for master

* fix: test workflow run on push only for master

* fix: remove parallel for gen types

* fix: benchmark

* fix: benchmark imports

* fix: pnpm

* fix: types errors and pnpm

* fix: types

* fix: types

* refactor: parser

* fix(parser): tests

* fix: preset tests

* fix: react types

* fix: react type declarations

* fix: pnpm lock file

* fix: react preset types

* fix: lock file

* fix: vue2 types

* feat: dev container support

* fix: types

* fix: types

* refactor: rewrite pkg-task, add nx gen-types deps, fix react/render.ts

* refactor: types

* fix: types

* fix: rename gen-types to types

* fix: nx build order

* fix: nx reset

* fix: define nx deps explicit

* fix: build

* fix: nx

* fix: nx order build

* fix: nx deps

* fix: bbob cli tests

* fix: tests

* fix: cli tests and import

* fix: test cover

* fix: cli cover
This commit is contained in:
Nikolay Kost
2024-04-23 21:11:14 +02:00
committed by GitHub
parent 05246b2aea
commit 8797f7f363
149 changed files with 6102 additions and 3670 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/bbob-parser/src/Token.js → packages/bbob-parser/src/Token.ts
+41 -49
View File
@@ -5,10 +5,11 @@ import {
} from '@bbob/plugin-helper';
// type, value, line, row,
const TOKEN_TYPE_ID = 'type'; // 0;
const TOKEN_VALUE_ID = 'value'; // 1;
const TOKEN_COLUMN_ID = 'row'; // 2;
const TOKEN_LINE_ID = 'line'; // 3;
const TOKEN_TYPE_ID = 't'; // 0;
const TOKEN_VALUE_ID = 'v'; // 1;
const TOKEN_COLUMN_ID = 'r'; // 2;
const TOKEN_LINE_ID = 'l'; // 3;
const TOKEN_TYPE_WORD = 1; // 'word';
const TOKEN_TYPE_TAG = 2; // 'tag';
@@ -17,29 +18,19 @@ const TOKEN_TYPE_ATTR_VALUE = 4; // 'attr-value';
const TOKEN_TYPE_SPACE = 5; // 'space';
const TOKEN_TYPE_NEW_LINE = 6; // 'new-line';
/**
* @param {Token} token
* @returns {string}
*/
const getTokenValue = (token) => {
const getTokenValue = (token: Token) => {
if (token && typeof token[TOKEN_VALUE_ID] !== 'undefined') {
return token[TOKEN_VALUE_ID];
}
return '';
};
/**
* @param {Token}token
* @returns {number}
*/
const getTokenLine = (token) => (token && token[TOKEN_LINE_ID]) || 0;
const getTokenColumn = (token) => (token && token[TOKEN_COLUMN_ID]) || 0;
/**
* @param {Token} token
* @returns {boolean}
*/
const isTextToken = (token) => {
const getTokenLine = (token: Token) => (token && token[TOKEN_LINE_ID]) || 0;
const getTokenColumn = (token: Token) => (token && token[TOKEN_COLUMN_ID]) || 0;
const isTextToken = (token: Token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_SPACE
|| token[TOKEN_TYPE_ID] === TOKEN_TYPE_NEW_LINE
@@ -49,20 +40,19 @@ const isTextToken = (token) => {
return false;
};
/**
* @param {Token} token
* @returns {boolean}
*/
const isTagToken = (token) => {
const isTagToken = (token: Token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_TAG;
}
return false;
};
const isTagEnd = (token) => getTokenValue(token).charCodeAt(0) === SLASH.charCodeAt(0);
const isTagStart = (token) => !isTagEnd(token);
const isAttrNameToken = (token) => {
const isTagEnd = (token: Token) => getTokenValue(token).charCodeAt(0) === SLASH.charCodeAt(0);
const isTagStart = (token: Token) => !isTagEnd(token);
const isAttrNameToken = (token: Token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_ATTR_NAME;
}
@@ -70,11 +60,7 @@ const isAttrNameToken = (token) => {
return false;
};
/**
* @param {Token} token
* @returns {boolean}
*/
const isAttrValueToken = (token) => {
const isAttrValueToken = (token: Token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_ATTR_VALUE;
}
@@ -82,13 +68,13 @@ const isAttrValueToken = (token) => {
return false;
};
const getTagName = (token) => {
const getTagName = (token: Token) => {
const value = getTokenValue(token);
return isTagEnd(token) ? value.slice(1) : value;
};
const convertTagToText = (token) => {
const tokenToText = (token: Token) => {
let text = OPEN_BRAKET;
text += getTokenValue(token);
@@ -97,23 +83,29 @@ const convertTagToText = (token) => {
return text;
};
class Token {
/**
* @param {String} type
* @param {String} value
* @param line
* @param row
*/
constructor(type, value, line, row) {
this[TOKEN_TYPE_ID] = Number(type);
/**
* @export
* @class Token
*/
class Token<TokenValue = string> {
private t: number // type
private v: string // value
private l: number // line
private r: number // row
constructor(type?: number, value?: TokenValue, row: number = 0, col: number = 0) {
this[TOKEN_LINE_ID] = row;
this[TOKEN_COLUMN_ID] = col;
this[TOKEN_TYPE_ID] = type || 0;
this[TOKEN_VALUE_ID] = String(value);
this[TOKEN_LINE_ID] = Number(line);
this[TOKEN_COLUMN_ID] = Number(row);
}
get type() {
return this[TOKEN_TYPE_ID]
}
isEmpty() {
// eslint-disable-next-line no-restricted-globals
return isNaN(this[TOKEN_TYPE_ID]);
return this[TOKEN_TYPE_ID] === 0 || isNaN(this[TOKEN_TYPE_ID]);
}
isText() {
@@ -157,7 +149,7 @@ class Token {
}
toString() {
return convertTagToText(this);
return tokenToText(this);
}
}
packages/bbob-parser/src/index.js
-2
View File
@@ -1,2 +0,0 @@
export { TagNode } from '@bbob/plugin-helper';
export { default, parse } from './parse';
packages/bbob-parser/src/index.ts
+5
View File
@@ -0,0 +1,5 @@
export { TagNode } from '@bbob/plugin-helper';
export { default } from './parse';
export * from './parse';
export * from './lexer'
export * from './types'
packages/bbob-parser/src/lexer.js → packages/bbob-parser/src/lexer.ts
+29 -51
View File
@@ -14,36 +14,17 @@ import {
import {
Token, TYPE_ATTR_NAME, TYPE_ATTR_VALUE, TYPE_NEW_LINE, TYPE_SPACE, TYPE_TAG, TYPE_WORD,
} from './Token';
import { createCharGrabber, trimChar, unquote } from './utils';
import { CharGrabber, createCharGrabber, trimChar, unquote } from './utils';
import type { LexerOptions, LexerTokenizer } from "./types";
// for cases <!-- -->
const EM = '!';
/**
* Creates a Token entity class
* @param {Number} type
* @param {String} value
* @param {Number} r line number
* @param {Number} cl char number in line
*/
const createToken = (type, value, r = 0, cl = 0) => new Token(type, value, r, cl);
export function createTokenOfType(type: number, value: string, r = 0, cl = 0) {
return new Token(type, value, r, cl)
}
/**
* @typedef {Object} Lexer
* @property {Function} tokenize
* @property {Function} isTokenNested
*/
/**
* @param {String} buffer
* @param {Object} options
* @param {Function} options.onToken
* @param {String} options.openTag
* @param {String} options.closeTag
* @param {Boolean} options.enableEscapeTags
* @return {Lexer}
*/
function createLexer(buffer, options = {}) {
export function createLexer(buffer: string, options: LexerOptions = {}): LexerTokenizer {
const STATE_WORD = 0;
const STATE_TAG = 1;
const STATE_TAG_ATTRS = 2;
@@ -59,7 +40,7 @@ function createLexer(buffer, options = {}) {
let stateMode = STATE_WORD;
let tagMode = TAG_STATE_NAME;
let contextFreeTag = '';
const tokens = new Array(Math.floor(buffer.length));
const tokens = new Array<Token<string>>(Math.floor(buffer.length));
const openTag = options.openTag || OPEN_BRAKET;
const closeTag = options.closeTag || CLOSE_BRAKET;
const escapeTags = !!options.enableEscapeTags;
@@ -76,20 +57,20 @@ function createLexer(buffer, options = {}) {
const WHITESPACES = [SPACE, TAB];
const SPECIAL_CHARS = [EQ, SPACE, TAB];
const isCharReserved = (char) => (RESERVED_CHARS.indexOf(char) >= 0);
const isNewLine = (char) => char === N;
const isWhiteSpace = (char) => (WHITESPACES.indexOf(char) >= 0);
const isCharToken = (char) => (NOT_CHAR_TOKENS.indexOf(char) === -1);
const isSpecialChar = (char) => (SPECIAL_CHARS.indexOf(char) >= 0);
const isEscapableChar = (char) => (char === openTag || char === closeTag || char === BACKSLASH);
const isEscapeChar = (char) => char === BACKSLASH;
const isCharReserved = (char: string) => (RESERVED_CHARS.indexOf(char) >= 0);
const isNewLine = (char: string) => char === N;
const isWhiteSpace = (char: string) => (WHITESPACES.indexOf(char) >= 0);
const isCharToken = (char: string) => (NOT_CHAR_TOKENS.indexOf(char) === -1);
const isSpecialChar = (char: string) => (SPECIAL_CHARS.indexOf(char) >= 0);
const isEscapableChar = (char: string) => (char === openTag || char === closeTag || char === BACKSLASH);
const isEscapeChar = (char: string) => char === BACKSLASH;
const onSkip = () => {
col++;
};
const unq = (val) => unquote(trimChar(val, QUOTEMARK));
const unq = (val: string) => unquote(trimChar(val, QUOTEMARK));
const checkContextFreeMode = (name, isClosingTag) => {
const checkContextFreeMode = (name: string, isClosingTag?: boolean) => {
if (contextFreeTag !== '' && isClosingTag) {
contextFreeTag = '';
}
@@ -106,8 +87,8 @@ function createLexer(buffer, options = {}) {
* @param {Number} type
* @param {String} value
*/
function emitToken(type, value) {
const token = createToken(type, value, row, col);
function emitToken(type: number, value: string) {
const token = createTokenOfType(type, value, row, col);
onToken(token);
@@ -115,9 +96,9 @@ function createLexer(buffer, options = {}) {
tokens[tokenIndex] = token;
}
function nextTagState(tagChars, isSingleValueTag) {
function nextTagState(tagChars: CharGrabber, isSingleValueTag: boolean) {
if (tagMode === TAG_STATE_ATTR) {
const validAttrName = (char) => !(char === EQ || isWhiteSpace(char));
const validAttrName = (char: string) => !(char === EQ || isWhiteSpace(char));
const name = tagChars.grabWhile(validAttrName);
const isEnd = tagChars.isLast();
const isValue = tagChars.getCurr() !== EQ;
@@ -143,7 +124,7 @@ function createLexer(buffer, options = {}) {
if (tagMode === TAG_STATE_VALUE) {
let stateSpecial = false;
const validAttrValue = (char) => {
const validAttrValue = (char: string) => {
// const isEQ = char === EQ;
const isQM = char === QUOTEMARK;
const prevChar = tagChars.getPrev();
@@ -152,7 +133,7 @@ function createLexer(buffer, options = {}) {
const isNextEQ = nextChar === EQ;
const isWS = isWhiteSpace(char);
// const isPrevWS = isWhiteSpace(prevChar);
const isNextWS = isWhiteSpace(nextChar);
const isNextWS = nextChar && isWhiteSpace(nextChar);
if (stateSpecial && isSpecialChar(char)) {
return true;
@@ -167,7 +148,7 @@ function createLexer(buffer, options = {}) {
}
if (!isSingleValueTag) {
return isWS === false;
return !isWS;
// return (isEQ || isWS) === false;
}
@@ -186,7 +167,7 @@ function createLexer(buffer, options = {}) {
return TAG_STATE_ATTR;
}
const validName = (char) => !(char === EQ || isWhiteSpace(char) || tagChars.isLast());
const validName = (char: string) => !(char === EQ || isWhiteSpace(char) || tagChars.isLast());
const name = tagChars.grabWhile(validName);
emitToken(TYPE_TAG, name);
@@ -214,7 +195,7 @@ function createLexer(buffer, options = {}) {
const substr = chars.substrUntilChar(closeTag);
const hasInvalidChars = substr.length === 0 || substr.indexOf(openTag) >= 0;
if (isCharReserved(nextChar) || hasInvalidChars || chars.isLast()) {
if ((nextChar && isCharReserved(nextChar)) || hasInvalidChars || chars.isLast()) {
emitToken(TYPE_WORD, currChar);
return STATE_WORD;
@@ -304,7 +285,7 @@ function createLexer(buffer, options = {}) {
chars.skip(); // skip the \ without emitting anything
if (isEscapableChar(nextChar)) {
if (nextChar && isEscapableChar(nextChar)) {
chars.skip(); // skip past the [, ] or \ as well
emitToken(TYPE_WORD, nextChar);
@@ -317,7 +298,7 @@ function createLexer(buffer, options = {}) {
return STATE_WORD;
}
const isChar = (char) => isCharToken(char) && !isEscapeChar(char);
const isChar = (char: string) => isCharToken(char) && !isEscapeChar(char);
const word = chars.grabWhile(isChar);
@@ -356,7 +337,7 @@ function createLexer(buffer, options = {}) {
return tokens;
}
function isTokenNested(token) {
function isTokenNested(token: Token) {
const value = openTag + SLASH + token.getValue();
// potential bottleneck
return buffer.indexOf(value) > -1;
@@ -365,8 +346,5 @@ function createLexer(buffer, options = {}) {
return {
tokenize,
isTokenNested,
};
}
}
export const createTokenOfType = createToken;
export { createLexer };
packages/bbob-parser/src/lexer_old.js
-242
View File
@@ -1,242 +0,0 @@
/* eslint-disable no-plusplus,no-param-reassign */
import {
OPEN_BRAKET,
CLOSE_BRAKET,
QUOTEMARK,
BACKSLASH,
SLASH,
SPACE,
TAB,
EQ,
N,
} from '@bbob/plugin-helper';
import {
Token, TYPE_ATTR_NAME, TYPE_ATTR_VALUE, TYPE_NEW_LINE, TYPE_SPACE, TYPE_TAG, TYPE_WORD,
} from './Token';
import { createCharGrabber, trimChar, unquote } from './utils';
// for cases <!-- -->
const EM = '!';
/**
* Creates a Token entity class
* @param {String} type
* @param {String} value
* @param {Number} r line number
* @param {Number} cl char number in line
*/
const createToken = (type, value, r = 0, cl = 0) => new Token(type, value, r, cl);
/**
* @typedef {Object} Lexer
* @property {Function} tokenize
* @property {Function} isTokenNested
*/
/**
* @param {String} buffer
* @param {Object} options
* @param {Function} options.onToken
* @param {String} options.openTag
* @param {String} options.closeTag
* @param {Boolean} options.enableEscapeTags
* @return {Lexer}
*/
function createLexer(buffer, options = {}) {
let row = 0;
let col = 0;
let tokenIndex = -1;
const tokens = new Array(Math.floor(buffer.length));
const openTag = options.openTag || OPEN_BRAKET;
const closeTag = options.closeTag || CLOSE_BRAKET;
const escapeTags = options.enableEscapeTags;
const RESERVED_CHARS = [closeTag, openTag, QUOTEMARK, BACKSLASH, SPACE, TAB, EQ, N, EM];
const NOT_CHAR_TOKENS = [
// ...(options.enableEscapeTags ? [BACKSLASH] : []),
openTag, SPACE, TAB, N,
];
const WHITESPACES = [SPACE, TAB];
const SPECIAL_CHARS = [EQ, SPACE, TAB];
const isCharReserved = (char) => (RESERVED_CHARS.indexOf(char) >= 0);
const isWhiteSpace = (char) => (WHITESPACES.indexOf(char) >= 0);
const isCharToken = (char) => (NOT_CHAR_TOKENS.indexOf(char) === -1);
const isSpecialChar = (char) => (SPECIAL_CHARS.indexOf(char) >= 0);
const isEscapableChar = (char) => (char === openTag || char === closeTag || char === BACKSLASH);
const isEscapeChar = (char) => char === BACKSLASH;
/**
* Emits newly created token to subscriber
* @param token
*/
const emitToken = (token) => {
if (options.onToken) {
options.onToken(token);
}
tokenIndex += 1;
tokens[tokenIndex] = token;
};
/**
* Parses params inside [myTag---params goes here---]content[/myTag]
* @param str
* @returns {{tag: *, attrs: Array}}
*/
const parseAttrs = (str) => {
let tagName = null;
let skipSpecialChars = false;
const attrTokens = [];
const attrCharGrabber = createCharGrabber(str);
const validAttr = (char) => {
const isEQ = char === EQ;
const isWS = isWhiteSpace(char);
const prevChar = attrCharGrabber.getPrev();
const nextChar = attrCharGrabber.getNext();
const isPrevSLASH = prevChar === BACKSLASH;
const isTagNameEmpty = tagName === null;
if (isTagNameEmpty) {
return (isEQ || isWS || attrCharGrabber.isLast()) === false;
}
if (skipSpecialChars && isSpecialChar(char)) {
return true;
}
if (char === QUOTEMARK && !isPrevSLASH) {
skipSpecialChars = !skipSpecialChars;
if (!skipSpecialChars && !(nextChar === EQ || isWhiteSpace(nextChar))) {
return false;
}
}
return (isEQ || isWS) === false;
};
const nextAttr = () => {
const attrStr = attrCharGrabber.grabWhile(validAttr);
const currChar = attrCharGrabber.getCurr();
// first string before space is a tag name [tagName params...]
if (tagName === null) {
tagName = attrStr;
} else if (isWhiteSpace(currChar) || currChar === QUOTEMARK || !attrCharGrabber.hasNext()) {
const escaped = unquote(trimChar(attrStr, QUOTEMARK));
attrTokens.push(createToken(TYPE_ATTR_VALUE, escaped, row, col));
} else {
attrTokens.push(createToken(TYPE_ATTR_NAME, attrStr, row, col));
}
attrCharGrabber.skip();
};
while (attrCharGrabber.hasNext()) {
nextAttr();
}
return { tag: tagName, attrs: attrTokens };
};
const bufferGrabber = createCharGrabber(buffer, {
onSkip: () => {
col++;
},
});
const next = () => {
const currChar = bufferGrabber.getCurr();
const nextChar = bufferGrabber.getNext();
if (currChar === N) {
bufferGrabber.skip();
col = 0;
row++;
emitToken(createToken(TYPE_NEW_LINE, currChar, row, col));
} else if (isWhiteSpace(currChar)) {
const str = bufferGrabber.grabWhile(isWhiteSpace);
emitToken(createToken(TYPE_SPACE, str, row, col));
} else if (escapeTags && isEscapeChar(currChar) && isEscapableChar(nextChar)) {
bufferGrabber.skip(); // skip the \ without emitting anything
bufferGrabber.skip(); // skip past the [, ] or \ as well
emitToken(createToken(TYPE_WORD, nextChar, row, col));
} else if (currChar === openTag) {
bufferGrabber.skip(); // skip openTag
// detect case where we have '[My word [tag][/tag]' or we have '[My last line word'
const substr = bufferGrabber.substrUntilChar(closeTag);
const hasInvalidChars = substr.length === 0 || substr.indexOf(openTag) >= 0;
if (isCharReserved(nextChar) || hasInvalidChars || bufferGrabber.isLast()) {
emitToken(createToken(TYPE_WORD, currChar, row, col));
} else {
const str = bufferGrabber.grabWhile((val) => val !== closeTag);
bufferGrabber.skip(); // skip closeTag
// [myTag ]
const isNoAttrsInTag = str.indexOf(EQ) === -1;
// [/myTag]
const isClosingTag = str[0] === SLASH;
if (isNoAttrsInTag || isClosingTag) {
emitToken(createToken(TYPE_TAG, str, row, col));
} else {
const parsed = parseAttrs(str);
emitToken(createToken(TYPE_TAG, parsed.tag, row, col));
parsed.attrs.map(emitToken);
}
}
} else if (currChar === closeTag) {
bufferGrabber.skip(); // skip closeTag
emitToken(createToken(TYPE_WORD, currChar, row, col));
} else if (isCharToken(currChar)) {
if (escapeTags && isEscapeChar(currChar) && !isEscapableChar(nextChar)) {
bufferGrabber.skip();
emitToken(createToken(TYPE_WORD, currChar, row, col));
} else {
const str = bufferGrabber.grabWhile((char) => {
if (escapeTags) {
return isCharToken(char) && !isEscapeChar(char);
}
return isCharToken(char);
});
emitToken(createToken(TYPE_WORD, str, row, col));
}
}
};
const tokenize = () => {
while (bufferGrabber.hasNext()) {
next();
}
tokens.length = tokenIndex + 1;
return tokens;
};
const isTokenNested = (token) => {
const value = openTag + SLASH + token.getValue();
// potential bottleneck
return buffer.indexOf(value) > -1;
};
return {
tokenize,
isTokenNested,
};
}
export const createTokenOfType = createToken;
export { createLexer };
packages/bbob-parser/src/parse.js
-303
View File
@@ -1,303 +0,0 @@
import {
TagNode, CLOSE_BRAKET, OPEN_BRAKET, isTagNode,
} from '@bbob/plugin-helper';
import { createLexer } from './lexer';
import { createList } from './utils';
/**
* @public
* @param {string} input
* @param {Object} opts
* @param {Function} opts.createTokenizer
* @param {Array<string>} opts.onlyAllowTags
* @param {Array<string>} opts.contextFreeTags
* @param {Boolean} opts.enableEscapeTags
* @param {string} opts.openTag
* @param {string} opts.closeTag
* @return {Array<string|TagNode>}
*/
const parse = (input, opts = {}) => {
const options = opts;
const openTag = options.openTag || OPEN_BRAKET;
const closeTag = options.closeTag || CLOSE_BRAKET;
const onlyAllowTags = (options.onlyAllowTags || [])
.filter(Boolean)
.map((tag) => tag.toLowerCase());
let tokenizer = null;
/**
* Result AST of nodes
* @private
* @type {NodeList}
*/
const nodes = createList();
/**
* Temp buffer of nodes that's nested to another node
* @private
* @type {NodeList}
*/
const nestedNodes = createList();
/**
* Temp buffer of nodes [tag..]...[/tag]
* @private
* @type {NodeList}
*/
const tagNodes = createList();
/**
* Temp buffer of tag attributes
* @private
* @type {NodeList}
*/
const tagNodesAttrName = createList();
/**
* Cache for nested tags checks
* @type Set<string>
*/
const nestedTagsMap = new Set();
/**
* @param {Token} token
* @returns {boolean}
*/
const isTokenNested = (token) => {
const value = token.getValue();
if (!nestedTagsMap.has(value) && tokenizer.isTokenNested && tokenizer.isTokenNested(token)) {
nestedTagsMap.add(value);
return true;
}
return nestedTagsMap.has(value);
};
/**
* @private
* @param {string} tagName
* @returns {boolean}
*/
const isTagNested = (tagName) => Boolean(nestedTagsMap.has(tagName));
/**
* @private
* @param {string} value
* @return {boolean}
*/
const isAllowedTag = (value) => {
if (onlyAllowTags.length) {
return onlyAllowTags.indexOf(value.toLowerCase()) >= 0;
}
return true;
};
/**
* Flushes temp tag nodes and its attributes buffers
* @private
* @return {Array}
*/
const flushTagNodes = () => {
if (tagNodes.flushLast()) {
tagNodesAttrName.flushLast();
}
};
/**
* @private
* @return {Array}
*/
const getNodes = () => {
const lastNestedNode = nestedNodes.getLast();
if (lastNestedNode && Array.isArray(lastNestedNode.content)) {
return lastNestedNode.content;
}
return nodes.toArray();
};
/**
* @private
* @param {string|TagNode} node
* @param {boolean} isNested
*/
const appendNodeAsString = (node, isNested = true) => {
const items = getNodes();
if (Array.isArray(items)) {
items.push(node.toTagStart({ openTag, closeTag }));
if (node.content.length) {
node.content.forEach((item) => {
items.push(item);
});
if (isNested) {
items.push(node.toTagEnd({ openTag, closeTag }));
}
}
}
};
/**
* @private
* @param {string|TagNode} node
*/
const appendNodes = (node) => {
const items = getNodes();
if (Array.isArray(items)) {
if (isTagNode(node)) {
if (isAllowedTag(node.tag)) {
items.push(node.toTagNode());
} else {
appendNodeAsString(node);
}
} else {
items.push(node);
}
}
};
/**
* @private
* @param {Token} token
*/
const handleTagStart = (token) => {
flushTagNodes();
const tagNode = TagNode.create(token.getValue());
const isNested = isTokenNested(token);
tagNodes.push(tagNode);
if (isNested) {
nestedNodes.push(tagNode);
} else {
appendNodes(tagNode, token);
}
};
/**
* @private
* @param {Token} token
*/
const handleTagEnd = (token) => {
flushTagNodes();
const lastNestedNode = nestedNodes.flushLast();
if (lastNestedNode) {
appendNodes(lastNestedNode, token);
} else if (typeof options.onError === 'function') {
const tag = token.getValue();
const line = token.getLine();
const column = token.getColumn();
options.onError({
message: `Inconsistent tag '${tag}' on line ${line} and column ${column}`,
tagName: tag,
lineNumber: line,
columnNumber: column,
});
}
};
/**
* @private
* @param {Token} token
*/
const handleTag = (token) => {
// [tag]
if (token.isStart()) {
handleTagStart(token);
}
// [/tag]
if (token.isEnd()) {
handleTagEnd(token);
}
};
/**
* @private
* @param {Token} token
*/
const handleNode = (token) => {
/**
* @type {TagNode}
*/
const lastTagNode = tagNodes.getLast();
const tokenValue = token.getValue();
const isNested = isTagNested(token);
if (lastTagNode) {
if (token.isAttrName()) {
tagNodesAttrName.push(tokenValue);
lastTagNode.attr(tagNodesAttrName.getLast(), '');
} else if (token.isAttrValue()) {
const attrName = tagNodesAttrName.getLast();
if (attrName) {
lastTagNode.attr(attrName, tokenValue);
tagNodesAttrName.flushLast();
} else {
lastTagNode.attr(tokenValue, tokenValue);
}
} else if (token.isText()) {
if (isNested) {
lastTagNode.append(tokenValue);
} else {
appendNodes(tokenValue);
}
} else if (token.isTag()) {
// if tag is not allowed, just past it as is
appendNodes(token.toString());
}
} else if (token.isText()) {
appendNodes(tokenValue);
} else if (token.isTag()) {
// if tag is not allowed, just past it as is
appendNodes(token.toString());
}
};
/**
* @private
* @param {Token} token
*/
const onToken = (token) => {
if (token.isTag()) {
handleTag(token);
} else {
handleNode(token);
}
};
tokenizer = (opts.createTokenizer ? opts.createTokenizer : createLexer)(input, {
onToken,
openTag,
closeTag,
onlyAllowTags: options.onlyAllowTags,
contextFreeTags: options.contextFreeTags,
enableEscapeTags: options.enableEscapeTags,
});
// eslint-disable-next-line no-unused-vars
const tokens = tokenizer.tokenize();
// handles situations where we open tag, but forgot close them
// for ex [q]test[/q][u]some[/u][q]some [u]some[/u] // forgot to close [/q]
// so we need to flush nested content to nodes array
const lastNestedNode = nestedNodes.flushLast();
if (lastNestedNode && isTagNested(lastNestedNode.tag)) {
appendNodeAsString(lastNestedNode, false);
}
return nodes.toArray();
};
export { parse };
export default parse;
packages/bbob-parser/src/parse.ts
+349
View File
@@ -0,0 +1,349 @@
import {
CLOSE_BRAKET,
OPEN_BRAKET,
TagNode,
isTagNode,
} from "@bbob/plugin-helper";
import { createLexer } from "./lexer";
import type { NodeContent, TagNodeTree } from "@bbob/plugin-helper";
import type { LexerTokenizer, LexerOptions } from "./types";
import type { Token } from "./Token";
type ParseError = {
tagName: string;
lineNumber: number;
columnNumber: number;
};
export interface ParseOptions {
createTokenizer?: (input: string, options?: LexerOptions) => LexerTokenizer;
openTag?: string;
closeTag?: string;
onlyAllowTags?: string[];
contextFreeTags?: string[];
enableEscapeTags?: boolean;
onError?: (error: ParseError) => void;
}
class NodeList<Value> {
private n: Value[];
constructor() {
this.n = [];
}
last() {
if (
Array.isArray(this.n) &&
this.n.length > 0 &&
typeof this.n[this.n.length - 1] !== "undefined"
) {
return this.n[this.n.length - 1];
}
return null;
}
flush() {
return this.n.length ? this.n.pop() : false;
}
push(value: Value) {
this.n.push(value);
}
toArray() {
return this.n;
}
}
const createList = <Type>() => new NodeList<Type>();
function parse(input: string, opts: ParseOptions = {}) {
const options = opts;
const openTag = options.openTag || OPEN_BRAKET;
const closeTag = options.closeTag || CLOSE_BRAKET;
const onlyAllowTags = (options.onlyAllowTags || [])
.filter(Boolean)
.map((tag) => tag.toLowerCase());
let tokenizer: LexerTokenizer | null = null;
/**
* Result AST of nodes
* @private
* @type {NodeList}
*/
const nodes = createList<TagNode>();
/**
* Temp buffer of nodes that's nested to another node
* @private
*/
const nestedNodes = createList<NodeContent>();
/**
* Temp buffer of nodes [tag..]...[/tag]
* @private
* @type {NodeList}
*/
const tagNodes = createList<TagNode>();
/**
* Temp buffer of tag attributes
* @private
* @type {NodeList}
*/
const tagNodesAttrName = createList<string>();
/**
* Cache for nested tags checks
*/
const nestedTagsMap = new Set<string>();
function isTokenNested(token: Token) {
const value = token.getValue();
const { isTokenNested } = tokenizer || {};
if (!nestedTagsMap.has(value) && isTokenNested && isTokenNested(token)) {
nestedTagsMap.add(value);
return true;
}
return nestedTagsMap.has(value);
}
/**
* @private
*/
function isTagNested(tagName: string) {
return Boolean(nestedTagsMap.has(tagName));
}
/**
* @private
*/
function isAllowedTag(value: string) {
if (onlyAllowTags.length) {
return onlyAllowTags.indexOf(value.toLowerCase()) >= 0;
}
return true;
}
/**
* Flushes temp tag nodes and its attributes buffers
* @private
*/
function flushTagNodes() {
if (tagNodes.flush()) {
tagNodesAttrName.flush();
}
}
/**
* @private
*/
function getNodes() {
const lastNestedNode = nestedNodes.last();
if (lastNestedNode && isTagNode(lastNestedNode)) {
return lastNestedNode.content;
}
return nodes.toArray();
}
/**
* @private
*/
function appendNodeAsString(
nodes?: TagNodeTree,
node?: TagNode,
isNested = true
) {
if (Array.isArray(nodes) && typeof node !== "undefined") {
nodes.push(node.toTagStart({ openTag, closeTag }));
if (Array.isArray(node.content) && node.content.length) {
node.content.forEach((item) => {
nodes.push(item);
});
if (isNested) {
nodes.push(node.toTagEnd({ openTag, closeTag }));
}
}
}
}
/**
* @private
*/
function appendNodes(nodes?: TagNodeTree, node?: NodeContent) {
if (Array.isArray(nodes) && typeof node !== "undefined") {
if (isTagNode(node)) {
if (isAllowedTag(node.tag)) {
nodes.push(node.toTagNode());
} else {
appendNodeAsString(nodes, node);
}
} else {
nodes.push(node);
}
}
}
/**
* @private
* @param {Token} token
*/
function handleTagStart(token: Token) {
flushTagNodes();
const tagNode = TagNode.create(token.getValue());
const isNested = isTokenNested(token);
tagNodes.push(tagNode);
if (isNested) {
nestedNodes.push(tagNode);
} else {
const nodes = getNodes();
appendNodes(nodes, tagNode);
}
}
/**
* @private
* @param {Token} token
*/
function handleTagEnd(token: Token) {
flushTagNodes();
const lastNestedNode = nestedNodes.flush();
if (lastNestedNode) {
const nodes = getNodes();
appendNodes(nodes, lastNestedNode);
} else if (typeof options.onError === "function") {
const tag = token.getValue();
const line = token.getLine();
const column = token.getColumn();
options.onError({
tagName: tag,
lineNumber: line,
columnNumber: column,
});
}
}
/**
* @private
* @param {Token} token
*/
function handleTag(token: Token) {
// [tag]
if (token.isStart()) {
handleTagStart(token);
}
// [/tag]
if (token.isEnd()) {
handleTagEnd(token);
}
}
/**
* @private
* @param {Token} token
*/
function handleNode(token: Token) {
/**
* @type {TagNode}
*/
const activeTagNode = tagNodes.last();
const tokenValue = token.getValue();
const isNested = isTagNested(token.toString());
const nodes = getNodes();
if (activeTagNode !== null) {
if (token.isAttrName()) {
tagNodesAttrName.push(tokenValue);
const attrName = tagNodesAttrName.last();
if (attrName) {
activeTagNode.attr(attrName, "");
}
} else if (token.isAttrValue()) {
const attrName = tagNodesAttrName.last();
if (attrName) {
activeTagNode.attr(attrName, tokenValue);
tagNodesAttrName.flush();
} else {
activeTagNode.attr(tokenValue, tokenValue);
}
} else if (token.isText()) {
if (isNested) {
activeTagNode.append(tokenValue);
} else {
appendNodes(nodes, tokenValue);
}
} else if (token.isTag()) {
// if tag is not allowed, just pass it as is
appendNodes(nodes, token.toString());
}
} else if (token.isText()) {
appendNodes(nodes, tokenValue);
} else if (token.isTag()) {
// if tag is not allowed, just pass it as is
appendNodes(nodes, token.toString());
}
}
/**
* @private
* @param {Token} token
*/
function onToken(token: Token) {
if (token.isTag()) {
handleTag(token);
} else {
handleNode(token);
}
}
const lexer = opts.createTokenizer ? opts.createTokenizer : createLexer;
tokenizer = lexer(input, {
onToken,
openTag,
closeTag,
onlyAllowTags: options.onlyAllowTags,
contextFreeTags: options.contextFreeTags,
enableEscapeTags: options.enableEscapeTags,
});
// eslint-disable-next-line no-unused-vars
const tokens = tokenizer.tokenize();
// handles situations where we open tag, but forgot close them
// for ex [q]test[/q][u]some[/u][q]some [u]some[/u] // forgot to close [/q]
// so we need to flush nested content to nodes array
const lastNestedNode = nestedNodes.flush();
if (
lastNestedNode !== null &&
lastNestedNode &&
isTagNode(lastNestedNode) &&
isTagNested(lastNestedNode.tag)
) {
appendNodeAsString(getNodes(), lastNestedNode, false);
}
return nodes.toArray();
}
export { parse };
export default parse;
packages/bbob-parser/src/types.ts
+15
View File
@@ -0,0 +1,15 @@
import type Token from "./Token";
export interface LexerTokenizer {
tokenize: () => Token<string>[];
isTokenNested?: (token: Token<string>) => boolean;
}
export type LexerOptions = {
openTag?: string;
closeTag?: string;
onlyAllowTags?: string[];
enableEscapeTags?: boolean;
contextFreeTags?: string[];
onToken?: (token?: Token<string>) => void;
};
packages/bbob-parser/src/utils.js
-165
View File
@@ -1,165 +0,0 @@
import {
QUOTEMARK,
BACKSLASH,
} from '@bbob/plugin-helper';
function CharGrabber(source, options) {
const cursor = {
pos: 0,
len: source.length,
};
const substrUntilChar = (char) => {
const { pos } = cursor;
const idx = source.indexOf(char, pos);
return idx >= 0 ? source.substring(pos, idx) : '';
};
const includes = (val) => source.indexOf(val, cursor.pos) >= 0;
const hasNext = () => cursor.len > cursor.pos;
const isLast = () => cursor.pos === cursor.len;
const skip = (num = 1, silent) => {
cursor.pos += num;
if (options && options.onSkip && !silent) {
options.onSkip();
}
};
const rest = () => source.substring(cursor.pos);
const grabN = (num = 0) => source.substring(cursor.pos, cursor.pos + num);
const curr = () => source[cursor.pos];
const prev = () => {
const prevPos = cursor.pos - 1;
return typeof source[prevPos] !== 'undefined' ? source[prevPos] : null;
};
const next = () => {
const nextPos = cursor.pos + 1;
return nextPos <= (source.length - 1) ? source[nextPos] : null;
};
const grabWhile = (cond, silent) => {
let start = 0;
if (hasNext()) {
start = cursor.pos;
while (hasNext() && cond(curr())) {
skip(1, silent);
}
}
return source.substring(start, cursor.pos);
};
/**
* @type {skip}
*/
this.skip = skip;
/**
* @returns {Boolean}
*/
this.hasNext = hasNext;
/**
* @returns {String}
*/
this.getCurr = curr;
/**
* @returns {String}
*/
this.getRest = rest;
/**
* @returns {String}
*/
this.getNext = next;
/**
* @returns {String}
*/
this.getPrev = prev;
/**
* @returns {Boolean}
*/
this.isLast = isLast;
/**
* @returns {Boolean}
*/
this.includes = includes;
/**
* @param {Function} cond
* @param {Boolean} silent
* @return {String}
*/
this.grabWhile = grabWhile;
/**
* @param {Number} num
* @return {String}
*/
this.grabN = grabN;
/**
* Grabs rest of string until it find a char
* @param {String} char
* @return {String}
*/
this.substrUntilChar = substrUntilChar;
}
/**
* Creates a grabber wrapper for source string, that helps to iterate over string char by char
* @param {String} source
* @param {Object} options
* @param {Function} options.onSkip
* @return CharGrabber
*/
export const createCharGrabber = (source, options) => new CharGrabber(source, options);
/**
* Trims string from start and end by char
* @example
* trimChar('*hello*', '*') ==> 'hello'
* @param {String} str
* @param {String} charToRemove
* @returns {String}
*/
export const trimChar = (str, charToRemove) => {
while (str.charAt(0) === charToRemove) {
// eslint-disable-next-line no-param-reassign
str = str.substring(1);
}
while (str.charAt(str.length - 1) === charToRemove) {
// eslint-disable-next-line no-param-reassign
str = str.substring(0, str.length - 1);
}
return str;
};
/**
* Unquotes \" to "
* @param str
* @return {String}
*/
export const unquote = (str) => str.replace(BACKSLASH + QUOTEMARK, QUOTEMARK);
function NodeList(values = []) {
const nodes = values;
const getLast = () => (
Array.isArray(nodes) && nodes.length > 0 && typeof nodes[nodes.length - 1] !== 'undefined'
? nodes[nodes.length - 1]
: null);
const flushLast = () => (nodes.length ? nodes.pop() : false);
const push = (value) => nodes.push(value);
const toArray = () => nodes;
this.push = push;
this.toArray = toArray;
this.getLast = getLast;
this.flushLast = flushLast;
}
/**
*
* @param values
* @return {NodeList}
*/
export const createList = (values = []) => new NodeList(values);
packages/bbob-parser/src/utils.ts
+121
View File
@@ -0,0 +1,121 @@
import {
QUOTEMARK,
BACKSLASH,
} from '@bbob/plugin-helper';
export type CharGrabberOptions = {
onSkip?: () => void
}
export class CharGrabber {
private s: string;
private c: { len: number; pos: number };
private o: CharGrabberOptions;
constructor(source: string, options: CharGrabberOptions = {}) {
this.s = source
this.c = {
pos: 0,
len: source.length,
};
this.o = options
}
skip(num = 1, silent?: boolean) {
this.c.pos += num;
if (this.o && this.o.onSkip && !silent) {
this.o.onSkip();
}
}
hasNext() {
return this.c.len > this.c.pos
}
getCurr() {
return this.s[this.c.pos]
}
getRest() {
return this.s.substring(this.c.pos)
}
getNext() {
const nextPos = this.c.pos + 1;
return nextPos <= (this.s.length - 1) ? this.s[nextPos] : null;
}
getPrev() {
const prevPos = this.c.pos - 1;
return typeof this.s[prevPos] !== 'undefined' ? this.s[prevPos] : null;
}
isLast() {
return this.c.pos === this.c.len
}
includes(val: string) {
return this.s.indexOf(val, this.c.pos) >= 0
}
grabWhile(condition: (curr: string) => boolean, silent?: boolean) {
let start = 0;
if (this.hasNext()) {
start = this.c.pos;
while (this.hasNext() && condition(this.getCurr())) {
this.skip(1, silent);
}
}
return this.s.substring(start, this.c.pos);
}
grabN(num: number = 0) {
return this.s.substring(this.c.pos, this.c.pos + num)
}
/**
* Grabs rest of string until it find a char
*/
substrUntilChar(char: string) {
const { pos } = this.c;
const idx = this.s.indexOf(char, pos);
return idx >= 0 ? this.s.substring(pos, idx) : '';
}
}
/**
* Creates a grabber wrapper for source string, that helps to iterate over string char by char
*/
export const createCharGrabber = (source: string, options?: CharGrabberOptions) => new CharGrabber(source, options);
/**
* Trims string from start and end by char
* @example
* trimChar('*hello*', '*') ==> 'hello'
*/
export const trimChar = (str: string, charToRemove: string) => {
while (str.charAt(0) === charToRemove) {
// eslint-disable-next-line no-param-reassign
str = str.substring(1);
}
while (str.charAt(str.length - 1) === charToRemove) {
// eslint-disable-next-line no-param-reassign
str = str.substring(0, str.length - 1);
}
return str;
};
/**
* Unquotes \" to "
*/
export const unquote = (str: string) => str.replace(BACKSLASH + QUOTEMARK, QUOTEMARK);