mirrors/bbob
2
0
Fork 0
mirror of https://github.com/tenrok/BBob.git synced 2026-06-20 20:00:33 +03:00
Code Activity

fix(289): contextFreeTags closing tag bug (#290)

Browse Source 
* feat: add tests

* fix: parsing context free

* refactor: code style

* chore: add changeset

* fix: disable coveralls
This commit is contained in:
Nikolay Kost
2025-09-14 23:18:41 +02:00
committed by GitHub
parent 0edd490a24
commit e943184294
9 changed files with 447 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/slimy-colts-drive.md
+20
View File
@@ -0,0 +1,20 @@
---
"@bbob/plugin-helper": patch
"@bbob/preset-html5": patch
"@bbob/preset-react": patch
"@bbob/preset-vue": patch
"@bbob/parser": patch
"@bbob/preset": patch
"@bbob/react": patch
"@bbob/types": patch
"@bbob/core": patch
"@bbob/html": patch
"@bbob/vue2": patch
"@bbob/vue3": patch
"@bbob/cli": patch
---
fixes problem with context free tags
now code like `[code][codeButton]text[/codeButton][/code]`
will be parsed correctly to `<code>[codeButton]text[/codeButton]</code>`
.github/workflows/test.yml
+2 -2
View File
@@ -40,5 +40,5 @@ jobs:
- name: Run the coverage - name: Run the coverage
run: pnpm run cover run: pnpm run cover
- name: Coveralls # - name: Coveralls
uses: coverallsapp/github-action@v2 # uses: coverallsapp/github-action@v2
packages/bbob-parser/src/Token.ts
+36 -50
View File
@@ -7,49 +7,48 @@ import type { Token as TokenInterface } from "@bbob/types";
// type, value, line, row, start pos, end pos // type, value, line, row, start pos, end pos
const TOKEN_TYPE_ID = 't'; // 0; export const TYPE_ID = 't'; // 0;
const TOKEN_VALUE_ID = 'v'; // 1; export const VALUE_ID = 'v'; // 1;
const TOKEN_COLUMN_ID = 'r'; // 2; export const LINE_ID = 'l'; // 3;
const TOKEN_LINE_ID = 'l'; // 3; export const COLUMN_ID = 'r'; // 2;
const TOKEN_START_POS_ID = 's'; // 4; export const START_POS_ID = 's'; // 4;
const TOKEN_END_POS_ID = 'e'; // 5; export const END_POS_ID = 'e'; // 5;
export const TYPE_WORD = 1; // 'word';
const TOKEN_TYPE_WORD = 1; // 'word'; export const TYPE_TAG = 2; // 'tag';
const TOKEN_TYPE_TAG = 2; // 'tag'; export const TYPE_ATTR_NAME = 3; // 'attr-name';
const TOKEN_TYPE_ATTR_NAME = 3; // 'attr-name'; export const TYPE_ATTR_VALUE = 4; // 'attr-value';
const TOKEN_TYPE_ATTR_VALUE = 4; // 'attr-value'; export const TYPE_SPACE = 5; // 'space';
const TOKEN_TYPE_SPACE = 5; // 'space'; export const TYPE_NEW_LINE = 6; // 'new-line';
const TOKEN_TYPE_NEW_LINE = 6; // 'new-line';
const getTokenValue = (token: Token) => { const getTokenValue = (token: Token) => {
if (token && typeof token[TOKEN_VALUE_ID] !== 'undefined') { if (token && typeof token[VALUE_ID] !== 'undefined') {
return token[TOKEN_VALUE_ID]; return token[VALUE_ID];
} }
return ''; return '';
}; };
const getTokenLine = (token: Token) => (token && token[TOKEN_LINE_ID]) || 0; const getTokenLine = (token: Token) => (token && token[LINE_ID]) || 0;
const getTokenColumn = (token: Token) => (token && token[TOKEN_COLUMN_ID]) || 0; const getTokenColumn = (token: Token) => (token && token[COLUMN_ID]) || 0;
const getStartPosition = (token: Token) => (token && token[TOKEN_START_POS_ID]) || 0; const getStartPosition = (token: Token) => (token && token[START_POS_ID]) || 0;
const getEndPosition = (token: Token) => (token && token[TOKEN_END_POS_ID]) || 0; const getEndPosition = (token: Token) => (token && token[END_POS_ID]) || 0;
const isTextToken = (token: Token) => { const isTextToken = (token: Token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') { if (token && typeof token[TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_SPACE return token[TYPE_ID] === TYPE_SPACE
|| token[TOKEN_TYPE_ID] === TOKEN_TYPE_NEW_LINE || token[TYPE_ID] === TYPE_NEW_LINE
|| token[TOKEN_TYPE_ID] === TOKEN_TYPE_WORD; || token[TYPE_ID] === TYPE_WORD;
} }
return false; return false;
}; };
const isTagToken = (token: Token) => { const isTagToken = (token: Token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') { if (token && typeof token[TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_TAG; return token[TYPE_ID] === TYPE_TAG;
} }
return false; return false;
@@ -60,16 +59,16 @@ const isTagEnd = (token: Token) => getTokenValue(token).charCodeAt(0) === SLASH.
const isTagStart = (token: Token) => !isTagEnd(token); const isTagStart = (token: Token) => !isTagEnd(token);
const isAttrNameToken = (token: Token) => { const isAttrNameToken = (token: Token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') { if (token && typeof token[TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_ATTR_NAME; return token[TYPE_ID] === TYPE_ATTR_NAME;
} }
return false; return false;
}; };
const isAttrValueToken = (token: Token) => { const isAttrValueToken = (token: Token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') { if (token && typeof token[TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_ATTR_VALUE; return token[TYPE_ID] === TYPE_ATTR_VALUE;
} }
return false; return false;
@@ -103,20 +102,20 @@ class Token<TokenValue = string> implements TokenInterface {
readonly e: number; // end pos readonly e: number; // end pos
constructor(type?: number, value?: TokenValue, row: number = 0, col: number = 0, start: number = 0, end: number = 0) { constructor(type?: number, value?: TokenValue, row: number = 0, col: number = 0, start: number = 0, end: number = 0) {
this[TOKEN_LINE_ID] = row; this[LINE_ID] = row;
this[TOKEN_COLUMN_ID] = col; this[COLUMN_ID] = col;
this[TOKEN_TYPE_ID] = type || 0; this[TYPE_ID] = type || 0;
this[TOKEN_VALUE_ID] = String(value); this[VALUE_ID] = String(value);
this[TOKEN_START_POS_ID] = start; this[START_POS_ID] = start;
this[TOKEN_END_POS_ID] = end; this[END_POS_ID] = end;
} }
get type() { get type() {
return this[TOKEN_TYPE_ID]; return this[TYPE_ID];
} }
isEmpty() { isEmpty() {
return this[TOKEN_TYPE_ID] === 0 || isNaN(this[TOKEN_TYPE_ID]); return this[TYPE_ID] === 0 || isNaN(this[TYPE_ID]);
} }
isText() { isText() {
@@ -172,18 +171,5 @@ class Token<TokenValue = string> implements TokenInterface {
} }
} }
export const TYPE_ID = TOKEN_TYPE_ID;
export const VALUE_ID = TOKEN_VALUE_ID;
export const LINE_ID = TOKEN_LINE_ID;
export const COLUMN_ID = TOKEN_COLUMN_ID;
export const START_POS_ID = TOKEN_START_POS_ID;
export const END_POS_ID = TOKEN_END_POS_ID;
export const TYPE_WORD = TOKEN_TYPE_WORD;
export const TYPE_TAG = TOKEN_TYPE_TAG;
export const TYPE_ATTR_NAME = TOKEN_TYPE_ATTR_NAME;
export const TYPE_ATTR_VALUE = TOKEN_TYPE_ATTR_VALUE;
export const TYPE_SPACE = TOKEN_TYPE_SPACE;
export const TYPE_NEW_LINE = TOKEN_TYPE_NEW_LINE;
export { Token }; export { Token };
export default Token; export default Token;
packages/bbob-parser/src/lexer.ts
+20 -14
View File
@@ -74,15 +74,18 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
col++; col++;
}; };
const checkContextFreeMode = (name: string, isClosingTag?: boolean) => { const setupContextFreeTag = (name: string, isClosingTag?: boolean) => {
if (contextFreeTag !== '' && isClosingTag) { if (contextFreeTag !== '' && isClosingTag) {
contextFreeTag = ''; contextFreeTag = '';
} }
if (contextFreeTag === '' && contextFreeTags.includes(name.toLowerCase())) { const tagName = name.toLowerCase()
contextFreeTag = name;
if (contextFreeTag === '' && isTokenNested(name) && contextFreeTags.includes(tagName)) {
contextFreeTag = tagName;
} }
}; };
const toEndTag = (tagName: string) => `${openTag}${SLASH}${tagName}${closeTag}`
const chars = createCharGrabber(buffer, { onSkip }); const chars = createCharGrabber(buffer, { onSkip });
@@ -178,12 +181,13 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
const name = tagChars.grabWhile(validName); const name = tagChars.grabWhile(validName);
emitToken(TYPE_TAG, name, start, masterStartPos + tagChars.getLength() + 1); emitToken(TYPE_TAG, name, start, masterStartPos + tagChars.getLength() + 1);
checkContextFreeMode(name);
setupContextFreeTag(name);
tagChars.skip(); tagChars.skip();
prevCol++; prevCol++;
// in cases when we has [url=someval]GET[/url] and we dont need to parse all // in cases when we have [url=someval]GET[/url] and we don't need to parse all
if (isSingleValueTag) { if (isSingleValueTag) {
return TAG_STATE_VALUE; return TAG_STATE_VALUE;
} }
@@ -202,7 +206,6 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
// detect case where we have '[My word [tag][/tag]' or we have '[My last line word' // detect case where we have '[My word [tag][/tag]' or we have '[My last line word'
const substr = chars.substrUntilChar(closeTag); const substr = chars.substrUntilChar(closeTag);
const hasInvalidChars = substr.length === 0 || substr.indexOf(openTag) >= 0; const hasInvalidChars = substr.length === 0 || substr.indexOf(openTag) >= 0;
const isNextCharReserved = nextChar && isCharReserved(nextChar) const isNextCharReserved = nextChar && isCharReserved(nextChar)
const isLastChar = chars.isLast() const isLastChar = chars.isLast()
@@ -228,7 +231,8 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
chars.skip(); // skip closeTag chars.skip(); // skip closeTag
emitToken(TYPE_TAG, name, startPos, endPos); emitToken(TYPE_TAG, name, startPos, endPos);
checkContextFreeMode(name, isClosingTag);
setupContextFreeTag(name, isClosingTag);
return STATE_WORD; return STATE_WORD;
} }
@@ -277,12 +281,11 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
if (chars.getCurr() === openTag) { if (chars.getCurr() === openTag) {
if (contextFreeTag) { if (contextFreeTag) {
const fullTagLen = openTag.length + SLASH.length + contextFreeTag.length; const fullTagName = toEndTag(contextFreeTag);
const fullTagName = `${openTag}${SLASH}${contextFreeTag}`; const foundTag = chars.grabN(fullTagName.length);
const foundTag = chars.grabN(fullTagLen); const isContextFreeEnded = foundTag.toLowerCase() === fullTagName.toLowerCase();
const isEndContextFreeMode = foundTag === fullTagName;
if (isEndContextFreeMode) { if (isContextFreeEnded) {
return STATE_TAG; return STATE_TAG;
} }
} else if (chars.includes(closeTag)) { } else if (chars.includes(closeTag)) {
@@ -357,12 +360,15 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
} }
function isTokenNested(tokenValue: string) { function isTokenNested(tokenValue: string) {
const value = openTag + SLASH + tokenValue; const value = toEndTag(tokenValue);
if (nestedMap.has(value)) { if (nestedMap.has(value)) {
return !!nestedMap.get(value); return !!nestedMap.get(value);
} else { } else {
const status = caseFreeTags ? (buffer.toLowerCase().indexOf(value.toLowerCase()) > -1) : (buffer.indexOf(value) > -1); const buf = caseFreeTags ? buffer.toLowerCase() : buffer;
const val = caseFreeTags ? value.toLowerCase() : value;
const status = buf.indexOf(val) > -1;
nestedMap.set(value, status); nestedMap.set(value, status);
packages/bbob-parser/src/parse.ts
+76 -78
View File
@@ -1,15 +1,10 @@
import type { NodeContent, TagNodeTree, LexerTokenizer, ParseOptions } from "@bbob/types"; import type { NodeContent, ParseOptions, TagNodeTree } from "@bbob/types";
import { import { CLOSE_BRAKET, isTagNode, OPEN_BRAKET, TagNode, } from "@bbob/plugin-helper";
CLOSE_BRAKET,
OPEN_BRAKET,
TagNode,
isTagNode,
} from "@bbob/plugin-helper";
import { createLexer } from "./lexer.js"; import { createLexer } from "./lexer.js";
import type { Token } from "./Token.js"; import { Token, TYPE_ATTR_NAME, TYPE_ATTR_VALUE, TYPE_NEW_LINE, TYPE_SPACE, TYPE_TAG, TYPE_WORD } from "./Token.js";
class NodeList<Value> { class NodeList<Value> {
private n: Value[]; private n: Value[];
@@ -85,15 +80,21 @@ function parse(input: string, opts: ParseOptions = {}) {
*/ */
const nestedTagsMap = new Set<string>(); const nestedTagsMap = new Set<string>();
function getValue(tokenValue: string) {
return caseFreeTags ? tokenValue.toLowerCase() : tokenValue;
}
function isTokenNested(token: Token) { function isTokenNested(token: Token) {
const tokenValue = token.getValue(); const tokenValue = token.getValue();
const value = caseFreeTags ? tokenValue.toLowerCase() : tokenValue; const value = getValue(tokenValue);
const { isTokenNested } = tokenizer || {}; const { isTokenNested } = tokenizer || {};
if (!nestedTagsMap.has(value) && isTokenNested && isTokenNested(value)) { if (!nestedTagsMap.has(value) && typeof isTokenNested === "function") {
nestedTagsMap.add(value); if (isTokenNested(value)) {
nestedTagsMap.add(value);
return true; return true;
}
} }
return nestedTagsMap.has(value); return nestedTagsMap.has(value);
@@ -103,13 +104,13 @@ function parse(input: string, opts: ParseOptions = {}) {
* @private * @private
*/ */
function isTagNested(tagName: string) { function isTagNested(tagName: string) {
return Boolean(nestedTagsMap.has(caseFreeTags ? tagName.toLowerCase() : tagName)); return Boolean(nestedTagsMap.has(getValue(tagName)));
} }
/** /**
* @private * @private
*/ */
function isAllowedTag(value: string) { function isTagAllowed(value: string) {
if (onlyAllowTags.length) { if (onlyAllowTags.length) {
return onlyAllowTags.indexOf(value.toLowerCase()) >= 0; return onlyAllowTags.indexOf(value.toLowerCase()) >= 0;
} }
@@ -121,7 +122,7 @@ function parse(input: string, opts: ParseOptions = {}) {
* Flushes temp tag nodes and its attributes buffers * Flushes temp tag nodes and its attributes buffers
* @private * @private
*/ */
function flushTagNodes() { function tagNodesFlush() {
if (tagNodes.flush()) { if (tagNodes.flush()) {
tagNodesAttrName.flush(); tagNodesAttrName.flush();
} }
@@ -143,7 +144,7 @@ function parse(input: string, opts: ParseOptions = {}) {
/** /**
* @private * @private
*/ */
function appendNodeAsString( function nodesAppendAsString(
nodes?: TagNodeTree, nodes?: TagNodeTree,
node?: TagNode, node?: TagNode,
isNested = true isNested = true
@@ -166,13 +167,13 @@ function parse(input: string, opts: ParseOptions = {}) {
/** /**
* @private * @private
*/ */
function appendNodes(nodes?: TagNodeTree, node?: NodeContent) { function nodesAppend(nodes?: TagNodeTree, node?: NodeContent) {
if (Array.isArray(nodes) && typeof node !== "undefined") { if (Array.isArray(nodes) && typeof node !== "undefined") {
if (isTagNode(node)) { if (isTagNode(node)) {
if (isAllowedTag(node.tag)) { if (isTagAllowed(node.tag)) {
nodes.push(node.toTagNode()); nodes.push(node.toTagNode());
} else { } else {
appendNodeAsString(nodes, node); nodesAppendAsString(nodes, node);
} }
} else { } else {
nodes.push(node); nodes.push(node);
@@ -184,8 +185,8 @@ function parse(input: string, opts: ParseOptions = {}) {
* @private * @private
* @param {Token} token * @param {Token} token
*/ */
function handleTagStart(token: Token) { function tagHandleStart(token: Token) {
flushTagNodes(); tagNodesFlush();
const tagNode = TagNode.create(token.getValue(), {}, [], { from: token.getStart(), to: token.getEnd() }); const tagNode = TagNode.create(token.getValue(), {}, [], { from: token.getStart(), to: token.getEnd() });
const isNested = isTokenNested(token); const isNested = isTokenNested(token);
@@ -196,7 +197,7 @@ function parse(input: string, opts: ParseOptions = {}) {
nestedNodes.push(tagNode); nestedNodes.push(tagNode);
} else { } else {
const nodes = getNodes(); const nodes = getNodes();
appendNodes(nodes, tagNode); nodesAppend(nodes, tagNode);
} }
} }
@@ -204,24 +205,24 @@ function parse(input: string, opts: ParseOptions = {}) {
* @private * @private
* @param {Token} token * @param {Token} token
*/ */
function handleTagEnd(token: Token) { function tagHandleEnd(token: Token) {
const tagName = token.getValue().slice(1); const tagName = token.getValue().slice(1);
const lastNestedNode = nestedNodes.flush(); const lastNestedNode = nestedNodes.flush();
flushTagNodes(); tagNodesFlush();
if (lastNestedNode) { if (lastNestedNode) {
const nodes = getNodes(); const nodes = getNodes()
if (isTagNode(lastNestedNode)) { if (isTagNode(lastNestedNode)) {
lastNestedNode.setEnd({ from: token.getStart(), to: token.getEnd() }); lastNestedNode.setEnd({ from: token.getStart(), to: token.getEnd() });
} }
appendNodes(nodes, lastNestedNode); nodesAppend(nodes, lastNestedNode);
} else if (!isTagNested(tagName)) { // when we have only close tag [/some] without any open tag } else if (!isTagNested(tagName)) { // when we have only close tag [/some] without any open tag
const nodes = getNodes(); const nodes = getNodes();
appendNodes(nodes, token.toString({ openTag, closeTag })); nodesAppend(nodes, token.toString({ openTag, closeTag }));
} else if (typeof options.onError === "function") { } else if (typeof options.onError === "function") {
const tag = token.getValue(); const tag = token.getValue();
const line = token.getLine(); const line = token.getLine();
@@ -239,23 +240,7 @@ function parse(input: string, opts: ParseOptions = {}) {
* @private * @private
* @param {Token} token * @param {Token} token
*/ */
function handleTag(token: Token) { function nodeHandle(token: Token) {
// [tag]
if (token.isStart()) {
handleTagStart(token);
}
// [/tag]
if (token.isEnd()) {
handleTagEnd(token);
}
}
/**
* @private
* @param {Token} token
*/
function handleNode(token: Token) {
/** /**
* @type {TagNode} * @type {TagNode}
*/ */
@@ -265,37 +250,47 @@ function parse(input: string, opts: ParseOptions = {}) {
const nodes = getNodes(); const nodes = getNodes();
if (activeTagNode !== null) { if (activeTagNode !== null) {
if (token.isAttrName()) { switch (token.type) {
tagNodesAttrName.push(tokenValue); case TYPE_ATTR_NAME:
const attrName = tagNodesAttrName.last(); tagNodesAttrName.push(tokenValue);
const attrName = tagNodesAttrName.last();
if (attrName) { if (attrName) {
activeTagNode.attr(attrName, ""); activeTagNode.attr(attrName, "");
} }
} else if (token.isAttrValue()) { break;
const attrName = tagNodesAttrName.last();
if (attrName) { case TYPE_ATTR_VALUE:
activeTagNode.attr(attrName, tokenValue); const attrValName = tagNodesAttrName.last();
tagNodesAttrName.flush();
} else { if (attrValName) {
activeTagNode.attr(tokenValue, tokenValue); activeTagNode.attr(attrValName, tokenValue);
} tagNodesAttrName.flush();
} else if (token.isText()) { } else {
if (isNested) { activeTagNode.attr(tokenValue, tokenValue);
activeTagNode.append(tokenValue); }
} else { break;
appendNodes(nodes, tokenValue);
} case TYPE_SPACE:
} else if (token.isTag()) { case TYPE_NEW_LINE:
// if tag is not allowed, just pass it as is case TYPE_WORD:
appendNodes(nodes, token.toString({ openTag, closeTag })); if (isNested) {
activeTagNode.append(tokenValue);
} else {
nodesAppend(nodes, tokenValue);
}
break;
case TYPE_TAG:
// if tag is not allowed, just pass it as is
nodesAppend(nodes, token.toString({ openTag, closeTag }));
break;
} }
} else if (token.isText()) { } else if (token.isText()) {
appendNodes(nodes, tokenValue); nodesAppend(nodes, tokenValue);
} else if (token.isTag()) { } else if (token.isTag()) {
// if tag is not allowed, just pass it as is // if tag is not allowed, just pass it as is
appendNodes(nodes, token.toString({ openTag, closeTag })); nodesAppend(nodes, token.toString({ openTag, closeTag }));
} }
} }
@@ -305,9 +300,17 @@ function parse(input: string, opts: ParseOptions = {}) {
*/ */
function onToken(token: Token) { function onToken(token: Token) {
if (token.isTag()) { if (token.isTag()) {
handleTag(token); // [tag]
if (token.isStart()) {
tagHandleStart(token);
}
// [/tag]
if (token.isEnd()) {
tagHandleEnd(token);
}
} else { } else {
handleNode(token); nodeHandle(token);
} }
} }
@@ -331,13 +334,8 @@ function parse(input: string, opts: ParseOptions = {}) {
// for ex [q]test[/q][u]some[/u][q]some [u]some[/u] // forgot to close [/q] // for ex [q]test[/q][u]some[/u][q]some [u]some[/u] // forgot to close [/q]
// so we need to flush nested content to nodes array // so we need to flush nested content to nodes array
const lastNestedNode = nestedNodes.flush(); const lastNestedNode = nestedNodes.flush();
if ( if (isTagNode(lastNestedNode) && isTagNested(lastNestedNode.tag)) {
lastNestedNode !== null && nodesAppendAsString(getNodes(), lastNestedNode, false);
lastNestedNode &&
isTagNode(lastNestedNode) &&
isTagNested(lastNestedNode.tag)
) {
appendNodeAsString(getNodes(), lastNestedNode, false);
} }
return nodes.toArray(); return nodes.toArray();
packages/bbob-parser/test/lexer.test.ts
+107 -108
View File
@@ -1,13 +1,6 @@
import { TYPE_ID, VALUE_ID, TYPE_WORD, TYPE_TAG, TYPE_ATTR_NAME, TYPE_ATTR_VALUE, TYPE_SPACE, TYPE_NEW_LINE, LINE_ID, COLUMN_ID, START_POS_ID, END_POS_ID } from '../src/Token'; import { TYPE_ID, VALUE_ID, TYPE_WORD, TYPE_TAG, TYPE_ATTR_NAME, TYPE_ATTR_VALUE, TYPE_SPACE, TYPE_NEW_LINE, LINE_ID, COLUMN_ID, START_POS_ID, END_POS_ID } from '../src/Token';
import { createLexer } from '../src/lexer'; import { createLexer } from '../src/lexer';
import { parse } from "../src";
declare global {
namespace jest {
interface Matchers<R> {
toBeMantchOutput(expected: Array<unknown>): CustomMatcherResult;
}
}
}
const TYPE = { const TYPE = {
WORD: TYPE_WORD, WORD: TYPE_WORD,
@@ -24,88 +17,94 @@ const tokenize = (input: string) => (createLexer(input).tokenize());
const tokenizeEscape = (input: string) => (createLexer(input, { enableEscapeTags: true }).tokenize()); const tokenizeEscape = (input: string) => (createLexer(input, { enableEscapeTags: true }).tokenize());
const tokenizeContextFreeTags = (input: string, tags: string[] = []) => (createLexer(input, { contextFreeTags: tags }).tokenize()); const tokenizeContextFreeTags = (input: string, tags: string[] = []) => (createLexer(input, { contextFreeTags: tags }).tokenize());
describe('lexer', () => { declare global {
namespace jest {
interface Matchers<R> {
toBeMatchOutput(expected: Array<unknown>): CustomMatcherResult;
}
}
}
expect.extend({
toBeMatchOutput(tokens, output) {
if (tokens.length !== output.length) {
return {
message: () =>
`expected tokens length ${tokens.length} to be ${output.length}`,
pass: false,
};
}
expect.extend({ for (let idx = 0; idx < tokens.length; idx++) {
toBeMantchOutput(tokens, output) { const token = tokens[idx];
if (tokens.length !== output.length) { const [type, value, col, row, startPos, endPos] = output[idx];
if (typeof token !== 'object') {
return { return {
message: () => message: () =>
`expected tokens length ${tokens.length} to be ${output.length}`, `token must to be Object`,
pass: false, pass: false,
}; };
} }
for (let idx = 0; idx < tokens.length; idx++) { if (token[TYPE_ID] !== type) {
const token = tokens[idx]; return {
const [type, value, col, row, startPos, endPos] = output[idx]; message: () =>
if (typeof token !== 'object') {
return {
message: () =>
`token must to be Object`,
pass: false,
};
}
if (token[TYPE_ID] !== type) {
return {
message: () =>
`expected token type ${TYPE_NAMES[type]} but received ${TYPE_NAMES[token[TYPE_ID]]} for ${JSON.stringify(output[idx])}`, `expected token type ${TYPE_NAMES[type]} but received ${TYPE_NAMES[token[TYPE_ID]]} for ${JSON.stringify(output[idx])}`,
pass: false, pass: false,
}; };
}
if (token[VALUE_ID] !== value) {
return {
message: () =>
`expected token value ${value} but received ${token[VALUE_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (token[LINE_ID] !== row) {
return {
message: () =>
`expected token row ${row} but received ${token[LINE_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (token[COLUMN_ID] !== col) {
return {
message: () =>
`expected token col ${col} but received ${token[COLUMN_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (type === TYPE.TAG && token[START_POS_ID] !== startPos) {
return {
message: () =>
`expected token start pos ${startPos} but received ${token[START_POS_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (type === TYPE.TAG && token[END_POS_ID] !== endPos) {
return {
message: () =>
`expected token end pos ${endPos} but received ${token[END_POS_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
} }
return { if (token[VALUE_ID] !== value) {
message: () => return {
`no valid output`, message: () =>
pass: true, `expected token value ${value} but received ${token[VALUE_ID]} for ${JSON.stringify(output[idx])}`,
}; pass: false,
}, };
}); }
if (token[LINE_ID] !== row) {
return {
message: () =>
`expected token row ${row} but received ${token[LINE_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (token[COLUMN_ID] !== col) {
return {
message: () =>
`expected token col ${col} but received ${token[COLUMN_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (type === TYPE.TAG && token[START_POS_ID] !== startPos) {
return {
message: () =>
`expected token start pos ${startPos} but received ${token[START_POS_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (type === TYPE.TAG && token[END_POS_ID] !== endPos) {
return {
message: () =>
`expected token end pos ${endPos} but received ${token[END_POS_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
}
return {
message: () =>
`no valid output`,
pass: true,
};
},
});
describe('lexer', () => {
test('single tag', () => { test('single tag', () => {
const input = '[SingleTag]'; const input = '[SingleTag]';
const tokens = tokenize(input); const tokens = tokenize(input);
@@ -113,7 +112,7 @@ describe('lexer', () => {
[TYPE.TAG, 'SingleTag', 0, 0, 0, 11], [TYPE.TAG, 'SingleTag', 0, 0, 0, 11],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('single tag with params', () => { test('single tag with params', () => {
@@ -124,7 +123,7 @@ describe('lexer', () => {
[TYPE.ATTR_VALUE, '111', 6, 0], [TYPE.ATTR_VALUE, '111', 6, 0],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('paired tag with single param', () => { test('paired tag with single param', () => {
@@ -137,7 +136,7 @@ describe('lexer', () => {
[TYPE.TAG, '/url', 17, 0, 16, 22], [TYPE.TAG, '/url', 17, 0, 16, 22],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('single fake tag', () => { test('single fake tag', () => {
@@ -149,7 +148,7 @@ describe('lexer', () => {
[TYPE.WORD, 'user=111]', 2, 0, 2], [TYPE.WORD, 'user=111]', 2, 0, 2],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('single tag with spaces', () => { test('single tag with spaces', () => {
@@ -160,7 +159,7 @@ describe('lexer', () => {
[TYPE.TAG, 'Single Tag', 0, 0, 0, 12], [TYPE.TAG, 'Single Tag', 0, 0, 0, 12],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
// @TODO: this is breaking change behavior // @TODO: this is breaking change behavior
@@ -175,7 +174,7 @@ describe('lexer', () => {
[TYPE.TAG, '/textarea', 25, 0], [TYPE.TAG, '/textarea', 25, 0],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('tags with single word and camel case params', () => { test('tags with single word and camel case params', () => {
@@ -213,7 +212,7 @@ describe('lexer', () => {
[TYPE.SPACE, ' ', 28, 2, 203], [TYPE.SPACE, ' ', 28, 2, 203],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('string with quotemarks', () => { test('string with quotemarks', () => {
@@ -232,7 +231,7 @@ describe('lexer', () => {
[TYPE.WORD, 'Adele', 22, 0], [TYPE.WORD, 'Adele', 22, 0],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('tags in brakets', () => { test('tags in brakets', () => {
@@ -249,7 +248,7 @@ describe('lexer', () => {
[TYPE.WORD, ']', 13, 0], [TYPE.WORD, ']', 13, 0],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('tag as param', () => { test('tag as param', () => {
@@ -262,7 +261,7 @@ describe('lexer', () => {
[TYPE.TAG, '/color', 21, 0, 21, 29], [TYPE.TAG, '/color', 21, 0, 21, 29],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('tag with quotemark params with spaces', () => { test('tag with quotemark params with spaces', () => {
@@ -278,7 +277,7 @@ describe('lexer', () => {
[TYPE.TAG, '/url', 42, 0, 42, 48], [TYPE.TAG, '/url', 42, 0, 42, 48],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('tag with escaped quotemark param', () => { test('tag with escaped quotemark param', () => {
@@ -292,7 +291,7 @@ describe('lexer', () => {
[TYPE.TAG, '/url', 26, 0, 26, 32], [TYPE.TAG, '/url', 26, 0, 26, 32],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('tag param without quotemarks', () => { test('tag param without quotemarks', () => {
@@ -306,7 +305,7 @@ describe('lexer', () => {
[TYPE.TAG, '/style', 26, 0, 25, 33], [TYPE.TAG, '/style', 26, 0, 25, 33],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('list tag with items', () => { test('list tag with items', () => {
@@ -344,7 +343,7 @@ describe('lexer', () => {
[TYPE.TAG, '/list', 0, 4, 52, 59], [TYPE.TAG, '/list', 0, 4, 52, 59],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('few tags without spaces', () => { test('few tags without spaces', () => {
@@ -366,7 +365,7 @@ describe('lexer', () => {
[TYPE.TAG, '/mytag3', 74, 0, 74, 83], [TYPE.TAG, '/mytag3', 74, 0, 74, 83],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('bad tags as texts', () => { test('bad tags as texts', () => {
@@ -434,7 +433,7 @@ describe('lexer', () => {
const tokens = tokenize(input); const tokens = tokenize(input);
const output = asserts[idx]; const output = asserts[idx];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
}); });
@@ -452,7 +451,7 @@ describe('lexer', () => {
[TYPE.TAG, 'Finger', 15, 0, 15, 23] [TYPE.TAG, 'Finger', 15, 0, 15, 23]
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('no close tag', () => { test('no close tag', () => {
@@ -467,7 +466,7 @@ describe('lexer', () => {
[TYPE.WORD, 'A', 13, 0], [TYPE.WORD, 'A', 13, 0],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('escaped tag', () => { test('escaped tag', () => {
@@ -482,7 +481,7 @@ describe('lexer', () => {
[TYPE.WORD, '[', 9, 0], [TYPE.WORD, '[', 9, 0],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('escaped tag and escaped backslash', () => { test('escaped tag and escaped backslash', () => {
@@ -502,7 +501,7 @@ describe('lexer', () => {
[TYPE.WORD, ']', 21, 0], [TYPE.WORD, ']', 21, 0],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('context free tag [code]', () => { test('context free tag [code]', () => {
@@ -520,12 +519,12 @@ describe('lexer', () => {
[TYPE.TAG, '/code', 25, 0, 25, 32], [TYPE.TAG, '/code', 25, 0, 25, 32],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('context free tag case insensitive [CODE]', () => { test('context free tag case insensitive [CODE]', () => {
const input = '[CODE] [b]some string[/b][/CODE]'; const tokens = tokenizeContextFreeTags('[CODE] [b]some string[/b][/CODE]', ['code']);
const tokens = tokenizeContextFreeTags(input, ['code']);
const output = [ const output = [
[TYPE.TAG, 'CODE', 0, 0, 0, 6], [TYPE.TAG, 'CODE', 0, 0, 0, 6],
[TYPE.SPACE, ' ', 6, 0], [TYPE.SPACE, ' ', 6, 0],
@@ -538,7 +537,7 @@ describe('lexer', () => {
[TYPE.TAG, '/CODE', 25, 0, 25, 32], [TYPE.TAG, '/CODE', 25, 0, 25, 32],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('bad closed tag with escaped backslash', () => { test('bad closed tag with escaped backslash', () => {
@@ -552,7 +551,7 @@ describe('lexer', () => {
[TYPE.WORD, 'b]', 9, 0], [TYPE.WORD, 'b]', 9, 0],
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
describe('html', () => { describe('html', () => {
@@ -575,7 +574,7 @@ describe('lexer', () => {
[TYPE.TAG, '/button', 78, 0, 78, 87] [TYPE.TAG, '/button', 78, 0, 78, 87]
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('attributes with no quotes or value', () => { test('attributes with no quotes or value', () => {
@@ -594,7 +593,7 @@ describe('lexer', () => {
[TYPE.TAG, '/button', 63, 0, 62, 71] [TYPE.TAG, '/button', 63, 0, 62, 71]
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test('attributes with no space between them. No valid, but accepted by the browser', () => { test('attributes with no space between them. No valid, but accepted by the browser', () => {
@@ -612,7 +611,7 @@ describe('lexer', () => {
[TYPE.TAG, '/button', 76, 0, 76, 85] [TYPE.TAG, '/button', 76, 0, 76, 85]
]; ];
expect(tokens).toBeMantchOutput(output); expect(tokens).toBeMatchOutput(output);
}); });
test.skip('style tag', () => { test.skip('style tag', () => {
@@ -634,7 +633,7 @@ input.buttonred{cursor:hand;font-family:verdana;background:#d12124;color:#fff;he
--> -->
</style>`; </style>`;
const tokens = tokenizeHTML(content); const tokens = tokenizeHTML(content);
expect(tokens).toBeMantchOutput([]); expect(tokens).toBeMatchOutput([]);
}); });
test.skip('script tag', () => { test.skip('script tag', () => {
@@ -645,7 +644,7 @@ input.buttonred{cursor:hand;font-family:verdana;background:#d12124;color:#fff;he
//--> //-->
</script>`; </script>`;
const tokens = tokenizeHTML(content); const tokens = tokenizeHTML(content);
expect(tokens).toBeMantchOutput([]); expect(tokens).toBeMatchOutput([]);
}); });
}); });
}); });
packages/bbob-parser/test/parse.test.ts
+153 -40
View File
@@ -1,12 +1,36 @@
import { parse } from '../src'; import { parse } from '../src';
import type { TagNode, TagNodeTree } from "@bbob/types"; import type { TagNode, TagNodeTree } from "@bbob/types";
describe('Parser', () => { const astToJSON = (ast: TagNodeTree) => Array.isArray(ast) ? ast.map(item => {
const expectOutput = (ast: TagNodeTree, output: Partial<TagNodeTree>) => { if (typeof item === 'object' && typeof item.toJSON === 'function') {
expect(ast).toBeInstanceOf(Array); return item.toJSON()
expect(ast).toMatchObject(output as {} | TagNode[]); }
};
return item
}) : ast
declare global {
namespace jest {
interface Matchers<R> {
toBeMatchAST(expected: Array<unknown>): CustomMatcherResult;
}
}
}
expect.extend({
toBeMatchAST(ast, output) {
expect(astToJSON(ast)).toMatchObject(output as {} | TagNode[]);
return {
message: () =>
`no valid output`,
pass: true,
};
},
});
describe('Parser', () => {
test('parse paired tags tokens', () => { test('parse paired tags tokens', () => {
const ast = parse('[best name=value]Foo Bar[/best]'); const ast = parse('[best name=value]Foo Bar[/best]');
const output = [ const output = [
@@ -31,7 +55,7 @@ describe('Parser', () => {
}, },
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
test('parse paired tags tokens 2', () => { test('parse paired tags tokens 2', () => {
@@ -56,7 +80,7 @@ describe('Parser', () => {
}, },
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
describe('onlyAllowTags', () => { describe('onlyAllowTags', () => {
@@ -87,7 +111,7 @@ describe('Parser', () => {
}, },
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
test('parse only allowed tags with params', () => { test('parse only allowed tags with params', () => {
@@ -96,7 +120,7 @@ describe('Parser', () => {
}; };
const ast = parse('hello [blah foo="bar"]world[/blah]', options); const ast = parse('hello [blah foo="bar"]world[/blah]', options);
expectOutput(ast, [ expect(ast).toBeMatchAST([
'hello', 'hello',
' ', ' ',
'[blah foo="bar"]', '[blah foo="bar"]',
@@ -111,7 +135,7 @@ describe('Parser', () => {
}; };
const ast = parse('hello [blah="bar"]world[/blah]', options); const ast = parse('hello [blah="bar"]world[/blah]', options);
expectOutput(ast, [ expect(ast).toBeMatchAST([
'hello', 'hello',
' ', ' ',
'[blah="bar"]', '[blah="bar"]',
@@ -180,7 +204,7 @@ describe('Parser', () => {
'[/tab]', '[/tab]',
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
test('parse only allowed tags case insensitive', () => { test('parse only allowed tags case insensitive', () => {
@@ -210,7 +234,7 @@ describe('Parser', () => {
}, },
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
}); });
@@ -243,8 +267,53 @@ describe('Parser', () => {
} }
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
test('nesting similar context free tags [code][codeButton]text[/codeButton][/code]', () => {
const ast = parse('[code][codeButton]text[/codeButton][/code]', {
contextFreeTags: ['code']
});
const output = [
{
tag: 'code',
attrs: {},
content: [
'[',
'codeButton]text',
'[',
'/codeButton]'
]
}
];
expect(ast).toBeMatchAST(output);
})
test('broken nesting similar context free tags [code][codeButton]text[/codeButton][code]', () => {
const ast = parse('[code][codeButton]text[/codeButton][code]', {
contextFreeTags: ['code']
});
const output = [
{
attrs: {},
content: [],
tag: 'code',
},
{
attrs: {},
content: ['text'],
tag: 'codeButton',
},
{
attrs: {},
content: [],
tag: 'code',
},
];
expect(ast).toBeMatchAST(output);
})
}); });
describe('caseFreeTags', () => { describe('caseFreeTags', () => {
@@ -268,7 +337,7 @@ describe('Parser', () => {
"[/H1]" "[/H1]"
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
test('case free tags', () => { test('case free tags', () => {
@@ -295,10 +364,57 @@ describe('Parser', () => {
} }
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
}) })
test('nesting similar tags [code][codeButton]text[/codeButton][/code]', () => {
const ast = parse('[code][codeButton]text[/codeButton][/code]');
const output = [
{
tag: 'code',
attrs: {},
content: [
{
tag: 'codeButton',
attrs: {},
content: [
'text'
]
}
]
}
];
expect(ast).toBeMatchAST(output);
})
test('forgot close code tag [code][codeButton]text[/codeButton][code]', () => {
const ast = parse('[code][codeButton]text[/codeButton][code]');
const output = [
{
tag: 'code',
attrs: {},
content: []
},
{
tag: 'codeButton',
attrs: {},
content: [
'text'
]
},
{
tag: 'code',
attrs: {},
content: []
}
];
expect(ast).toBeMatchAST(output);
})
test('parse inconsistent tags', () => { test('parse inconsistent tags', () => {
const ast = parse('[h1 name=value]Foo [Bar] /h1]'); const ast = parse('[h1 name=value]Foo [Bar] /h1]');
const output = [ const output = [
@@ -316,7 +432,7 @@ describe('Parser', () => {
'Foo', 'Foo',
' ', ' ',
{ {
tag: 'bar', tag: 'Bar',
attrs: {}, attrs: {},
content: [], content: [],
start: { start: {
@@ -328,7 +444,7 @@ describe('Parser', () => {
'/h1]', '/h1]',
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
test('parse closed tag', () => { test('parse closed tag', () => {
@@ -337,7 +453,7 @@ describe('Parser', () => {
'[/h1]', '[/h1]',
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
test('parse tag with value param', () => { test('parse tag with value param', () => {
@@ -360,7 +476,7 @@ describe('Parser', () => {
}, },
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
test('parse tag with quoted param with spaces', () => { test('parse tag with quoted param with spaces', () => {
@@ -385,7 +501,7 @@ describe('Parser', () => {
}, },
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
test('parse single tag with params', () => { test('parse single tag with params', () => {
@@ -404,7 +520,7 @@ describe('Parser', () => {
}, },
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
test('detect inconsistent tag', () => { test('detect inconsistent tag', () => {
@@ -463,14 +579,14 @@ describe('Parser', () => {
}, },
]; ];
expectOutput(ast, output); expect(ast).toBeMatchAST(output);
}); });
// @TODO: this is breaking change behavior // @TODO: this is breaking change behavior
test.skip('parse tags with single attributes like disabled', () => { test.skip('parse tags with single attributes like disabled', () => {
const ast = parse('[b]hello[/b] [textarea disabled]world[/textarea]'); const ast = parse('[b]hello[/b] [textarea disabled]world[/textarea]');
expectOutput(ast, [ expect(ast).toBeMatchAST([
{ {
tag: 'b', tag: 'b',
attrs: {}, attrs: {},
@@ -506,7 +622,7 @@ describe('Parser', () => {
test('parse url tag with get params', () => { test('parse url tag with get params', () => {
const ast = parse('[url=https://github.com/JiLiZART/bbob/search?q=any&unscoped_q=any]GET[/url]'); const ast = parse('[url=https://github.com/JiLiZART/bbob/search?q=any&unscoped_q=any]GET[/url]');
expectOutput(ast, [ expect(ast).toBeMatchAST([
{ {
tag: 'url', tag: 'url',
attrs: { attrs: {
@@ -531,7 +647,7 @@ describe('Parser', () => {
attr value"] this is a spoiler attr value"] this is a spoiler
[b]this is bold [i]this is bold and italic[/i] this is bold again[/b] [b]this is bold [i]this is bold and italic[/i] this is bold again[/b]
[/spoiler]this is outside again`); [/spoiler]this is outside again`);
expectOutput(ast, [ expect(ast).toBeMatchAST([
"this", "this",
" ", " ",
"is", "is",
@@ -632,7 +748,7 @@ describe('Parser', () => {
[avatar href="/avatar/4/3/b/1606.jpg@20x20?cache=1561462725&bgclr=ffffff" size=xs][/avatar] [avatar href="/avatar/4/3/b/1606.jpg@20x20?cache=1561462725&bgclr=ffffff" size=xs][/avatar]
Group Name Go[/url] `); Group Name Go[/url] `);
expectOutput(ast, [ expect(ast).toBeMatchAST([
{ {
tag: 'url', tag: 'url',
attrs: { attrs: {
@@ -684,7 +800,7 @@ describe('Parser', () => {
test('parse url tag with # and = symbols [google docs]', () => { test('parse url tag with # and = symbols [google docs]', () => {
const ast = parse('[url href=https://docs.google.com/spreadsheets/d/1W9VPUESF_NkbSa_HtRFrQNl0nYo8vPCxJFy7jD3Tpio/edit#gid=0]Docs[/url]'); const ast = parse('[url href=https://docs.google.com/spreadsheets/d/1W9VPUESF_NkbSa_HtRFrQNl0nYo8vPCxJFy7jD3Tpio/edit#gid=0]Docs[/url]');
expectOutput(ast, [ expect(ast).toBeMatchAST([
{ {
tag: 'url', tag: 'url',
attrs: { attrs: {
@@ -710,8 +826,7 @@ sdfasdfasdf
[url=xxx]xxx[/url]`; [url=xxx]xxx[/url]`;
expectOutput( expect(parse(str)).toBeMatchAST(
parse(str),
[ [
{ {
tag: 'quote', attrs: {}, content: ['some'], tag: 'quote', attrs: {}, content: ['some'],
@@ -760,8 +875,7 @@ sdfasdfasdf
test('parse with lost closing tag on from', () => { test('parse with lost closing tag on from', () => {
const str = `[quote]xxxsdfasdf[quote]some[/quote][color=red]test[/color]sdfasdfasdf[url=xxx]xxx[/url]`; const str = `[quote]xxxsdfasdf[quote]some[/quote][color=red]test[/color]sdfasdfasdf[url=xxx]xxx[/url]`;
expectOutput( expect(parse(str)).toBeMatchAST(
parse(str),
[ [
'[quote]', '[quote]',
'xxxsdfasdf', 'xxxsdfasdf',
@@ -806,8 +920,7 @@ sdfasdfasdf
test('parse with lost closing tag on to', () => { test('parse with lost closing tag on to', () => {
const str = `[quote]some[/quote][color=red]test[/color]sdfasdfasdf[url=xxx]xxx[/url][quote]xxxsdfasdf`; const str = `[quote]some[/quote][color=red]test[/color]sdfasdfasdf[url=xxx]xxx[/url][quote]xxxsdfasdf`;
expectOutput( expect(parse(str)).toBeMatchAST(
parse(str),
[ [
{ {
tag: 'quote', attrs: {}, content: ['some'], tag: 'quote', attrs: {}, content: ['some'],
@@ -852,7 +965,7 @@ sdfasdfasdf
test('parse with url in tag content', () => { test('parse with url in tag content', () => {
const input = parse('[img]https://tw.greywool.com/i/e3Ph5.png[/img]'); const input = parse('[img]https://tw.greywool.com/i/e3Ph5.png[/img]');
expectOutput(input, [ expect(input).toBeMatchAST([
{ {
tag: 'img', tag: 'img',
attrs: {}, attrs: {},
@@ -874,7 +987,7 @@ sdfasdfasdf
whitespaceInTags: false whitespaceInTags: false
}) })
expectOutput(input, [ expect(input).toBeMatchAST([
{ {
tag: 'b', tag: 'b',
attrs: {}, attrs: {},
@@ -913,7 +1026,7 @@ sdfasdfasdf
const content = `<button id="test0" class="value0" title="value1">class="value0" title="value1"</button>`; const content = `<button id="test0" class="value0" title="value1">class="value0" title="value1"</button>`;
const ast = parseHTML(content); const ast = parseHTML(content);
expectOutput(ast, [ expect(ast).toBeMatchAST([
{ {
"tag": "button", "tag": "button",
"attrs": { "attrs": {
@@ -942,7 +1055,7 @@ sdfasdfasdf
const content = `<button id="test1" class=value2 disabled required>class=value2 disabled</button>`; const content = `<button id="test1" class=value2 disabled required>class=value2 disabled</button>`;
const ast = parseHTML(content); const ast = parseHTML(content);
expectOutput(ast, [ expect(ast).toBeMatchAST([
{ {
"tag": "button", "tag": "button",
"attrs": { "attrs": {
@@ -972,7 +1085,7 @@ sdfasdfasdf
const content = `<button id="test2" class="value4"title="value5">class="value4"title="value5"</button>`; const content = `<button id="test2" class="value4"title="value5">class="value4"title="value5"</button>`;
const ast = parseHTML(content); const ast = parseHTML(content);
expectOutput(ast, [ expect(ast).toBeMatchAST([
{ {
"tag": "button", "tag": "button",
"attrs": { "attrs": {
@@ -1000,7 +1113,7 @@ sdfasdfasdf
enableEscapeTags: true enableEscapeTags: true
}); });
expectOutput(ast, [ expect(ast).toBeMatchAST([
'[', '[',
'b', 'b',
']', ']',
@@ -1016,7 +1129,7 @@ sdfasdfasdf
enableEscapeTags: true enableEscapeTags: true
}); });
expectOutput(ast, [ expect(ast).toBeMatchAST([
'\\', '\\',
'[', '[',
'b', 'b',
packages/bbob-plugin-helper/src/TagNode.ts
+32 -38
View File
@@ -1,14 +1,7 @@
import type { NodeContent, TagNodeObject, TagNodeTree, TagPosition } from "@bbob/types"; import type { NodeContent, TagNodeObject, TagNodeTree, TagPosition } from "@bbob/types";
import { OPEN_BRAKET, CLOSE_BRAKET, SLASH } from './char.js'; import { CLOSE_BRAKET, OPEN_BRAKET, SLASH } from './char.js';
import { import { appendToNode, attrsToString, attrValue, getNodeLength, getUniqAttr, isTagNode, } from './helpers.js';
getUniqAttr,
getNodeLength,
appendToNode,
attrsToString,
attrValue,
isTagNode,
} from './helpers.js';
const getTagAttrs = <AttrValue>(tag: string, params: Record<string, AttrValue>) => { const getTagAttrs = <AttrValue>(tag: string, params: Record<string, AttrValue>) => {
const uniqAttr = getUniqAttr(params); const uniqAttr = getUniqAttr(params);
@@ -27,19 +20,19 @@ const getTagAttrs = <AttrValue>(tag: string, params: Record<string, AttrValue>)
return `${tag}${attrsToString(params)}`; return `${tag}${attrsToString(params)}`;
}; };
const renderContent = (content: TagNodeTree, openTag: string, closeTag: string) => { const toString = (node: NodeContent, openTag: string, closeTag: string) => {
const toString = (node: NodeContent) => { if (isTagNode(node)) {
if (isTagNode(node)) { return node.toString({ openTag, closeTag });
return node.toString({ openTag, closeTag }); }
}
return String(node); return String(node);
}; };
const nodeTreeToString = (content: TagNodeTree, openTag: string, closeTag: string) => {
if (Array.isArray(content)) { if (Array.isArray(content)) {
return content.reduce<string>((r, node) => { return content.reduce<string>((r, node) => {
if (node !== null) { if (node !== null) {
return r + toString(node); return r + toString(node, openTag, closeTag);
} }
return r; return r;
@@ -47,7 +40,7 @@ const renderContent = (content: TagNodeTree, openTag: string, closeTag: string)
} }
if (content) { if (content) {
return toString(content); return toString(content, openTag, closeTag);
} }
return null; return null;
@@ -60,10 +53,16 @@ export class TagNode<TagValue extends any = any> implements TagNodeObject {
public start?: TagPosition; public start?: TagPosition;
public end?: TagPosition; public end?: TagPosition;
constructor(tag: string | TagValue, attrs: Record<string, unknown>, content: TagNodeTree) { constructor(tag: string | TagValue, attrs: Record<string, unknown>, content: TagNodeTree, start?: TagPosition, end?: TagPosition) {
this.tag = tag; this.tag = tag
this.attrs = attrs; this.attrs = attrs;
this.content = content; this.content = content;
this.start = start;
this.end = end;
}
get length(): number {
return getNodeLength(this);
} }
attr(name: string, value?: unknown) { attr(name: string, value?: unknown) {
@@ -86,10 +85,6 @@ export class TagNode<TagValue extends any = any> implements TagNodeObject {
this.end = value; this.end = value;
} }
get length(): number {
return getNodeLength(this);
}
toTagStart({ openTag = OPEN_BRAKET, closeTag = CLOSE_BRAKET } = {}) { toTagStart({ openTag = OPEN_BRAKET, closeTag = CLOSE_BRAKET } = {}) {
const tagAttrs = getTagAttrs(String(this.tag), this.attrs); const tagAttrs = getTagAttrs(String(this.tag), this.attrs);
@@ -101,18 +96,11 @@ export class TagNode<TagValue extends any = any> implements TagNodeObject {
} }
toTagNode() { toTagNode() {
const newNode = new TagNode(String(this.tag).toLowerCase(), this.attrs, this.content); return new TagNode(this.tag, this.attrs, this.content, this.start, this.end);
if (this.start) {
newNode.setStart(this.start);
}
if (this.end) {
newNode.setEnd(this.end);
}
return newNode;
} }
toString({ openTag = OPEN_BRAKET, closeTag = CLOSE_BRAKET } = {}): string { toString({ openTag = OPEN_BRAKET, closeTag = CLOSE_BRAKET } = {}): string {
const content = this.content ? renderContent(this.content, openTag, closeTag) : ''; const content = this.content ? nodeTreeToString(this.content, openTag, closeTag) : '';
const tagStart = this.toTagStart({ openTag, closeTag }); const tagStart = this.toTagStart({ openTag, closeTag });
if (this.content === null || Array.isArray(this.content) && this.content.length === 0) { if (this.content === null || Array.isArray(this.content) && this.content.length === 0) {
@@ -122,12 +110,18 @@ export class TagNode<TagValue extends any = any> implements TagNodeObject {
return `${tagStart}${content}${this.toTagEnd({ openTag, closeTag })}`; return `${tagStart}${content}${this.toTagEnd({ openTag, closeTag })}`;
} }
toJSON() {
return {
tag: this.tag,
attrs: this.attrs,
content: this.content,
start: this.start,
end: this.end,
};
}
static create(tag: string, attrs: Record<string, unknown> = {}, content: TagNodeTree = null, start?: TagPosition) { static create(tag: string, attrs: Record<string, unknown> = {}, content: TagNodeTree = null, start?: TagPosition) {
const node = new TagNode(tag, attrs, content); return new TagNode(tag, attrs, content, start);
if (start) {
node.setStart(start);
}
return node;
} }
static isOf(node: TagNode, type: string) { static isOf(node: TagNode, type: string) {
packages/bbob-types/src/common.ts
+1
View File
@@ -6,6 +6,7 @@ export interface TagNodeObject<TagValue extends any = any> {
content?: TagNodeTree<TagValue>; content?: TagNodeTree<TagValue>;
start?: TagPosition; start?: TagPosition;
end?: TagPosition; end?: TagPosition;
toJSON?: () => TagNodeObject<TagValue>;
} }
export type NodeContent<TagValue extends any = any> = TagNodeObject<TagValue> | StringNode | null; export type NodeContent<TagValue extends any = any> = TagNodeObject<TagValue> | StringNode | null;