2
0
mirror of https://github.com/tenrok/BBob.git synced 2026-06-11 18:02:26 +03:00

fix(289): contextFreeTags closing tag bug (#290)

* feat: add tests

* fix: parsing context free

* refactor: code style

* chore: add changeset

* fix: disable coveralls
This commit is contained in:
Nikolay Kost
2025-09-14 23:18:41 +02:00
committed by GitHub
parent 0edd490a24
commit e943184294
9 changed files with 447 additions and 330 deletions
+107 -108
View File
@@ -1,13 +1,6 @@
import { TYPE_ID, VALUE_ID, TYPE_WORD, TYPE_TAG, TYPE_ATTR_NAME, TYPE_ATTR_VALUE, TYPE_SPACE, TYPE_NEW_LINE, LINE_ID, COLUMN_ID, START_POS_ID, END_POS_ID } from '../src/Token';
import { createLexer } from '../src/lexer';
declare global {
namespace jest {
interface Matchers<R> {
toBeMantchOutput(expected: Array<unknown>): CustomMatcherResult;
}
}
}
import { parse } from "../src";
const TYPE = {
WORD: TYPE_WORD,
@@ -24,88 +17,94 @@ const tokenize = (input: string) => (createLexer(input).tokenize());
const tokenizeEscape = (input: string) => (createLexer(input, { enableEscapeTags: true }).tokenize());
const tokenizeContextFreeTags = (input: string, tags: string[] = []) => (createLexer(input, { contextFreeTags: tags }).tokenize());
describe('lexer', () => {
declare global {
namespace jest {
interface Matchers<R> {
toBeMatchOutput(expected: Array<unknown>): CustomMatcherResult;
}
}
}
expect.extend({
toBeMatchOutput(tokens, output) {
if (tokens.length !== output.length) {
return {
message: () =>
`expected tokens length ${tokens.length} to be ${output.length}`,
pass: false,
};
}
expect.extend({
toBeMantchOutput(tokens, output) {
if (tokens.length !== output.length) {
for (let idx = 0; idx < tokens.length; idx++) {
const token = tokens[idx];
const [type, value, col, row, startPos, endPos] = output[idx];
if (typeof token !== 'object') {
return {
message: () =>
`expected tokens length ${tokens.length} to be ${output.length}`,
`token must to be Object`,
pass: false,
};
}
for (let idx = 0; idx < tokens.length; idx++) {
const token = tokens[idx];
const [type, value, col, row, startPos, endPos] = output[idx];
if (typeof token !== 'object') {
return {
message: () =>
`token must to be Object`,
pass: false,
};
}
if (token[TYPE_ID] !== type) {
return {
message: () =>
if (token[TYPE_ID] !== type) {
return {
message: () =>
`expected token type ${TYPE_NAMES[type]} but received ${TYPE_NAMES[token[TYPE_ID]]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (token[VALUE_ID] !== value) {
return {
message: () =>
`expected token value ${value} but received ${token[VALUE_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (token[LINE_ID] !== row) {
return {
message: () =>
`expected token row ${row} but received ${token[LINE_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (token[COLUMN_ID] !== col) {
return {
message: () =>
`expected token col ${col} but received ${token[COLUMN_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (type === TYPE.TAG && token[START_POS_ID] !== startPos) {
return {
message: () =>
`expected token start pos ${startPos} but received ${token[START_POS_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (type === TYPE.TAG && token[END_POS_ID] !== endPos) {
return {
message: () =>
`expected token end pos ${endPos} but received ${token[END_POS_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
pass: false,
};
}
return {
message: () =>
`no valid output`,
pass: true,
};
},
});
if (token[VALUE_ID] !== value) {
return {
message: () =>
`expected token value ${value} but received ${token[VALUE_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (token[LINE_ID] !== row) {
return {
message: () =>
`expected token row ${row} but received ${token[LINE_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (token[COLUMN_ID] !== col) {
return {
message: () =>
`expected token col ${col} but received ${token[COLUMN_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (type === TYPE.TAG && token[START_POS_ID] !== startPos) {
return {
message: () =>
`expected token start pos ${startPos} but received ${token[START_POS_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (type === TYPE.TAG && token[END_POS_ID] !== endPos) {
return {
message: () =>
`expected token end pos ${endPos} but received ${token[END_POS_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
}
return {
message: () =>
`no valid output`,
pass: true,
};
},
});
describe('lexer', () => {
test('single tag', () => {
const input = '[SingleTag]';
const tokens = tokenize(input);
@@ -113,7 +112,7 @@ describe('lexer', () => {
[TYPE.TAG, 'SingleTag', 0, 0, 0, 11],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('single tag with params', () => {
@@ -124,7 +123,7 @@ describe('lexer', () => {
[TYPE.ATTR_VALUE, '111', 6, 0],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('paired tag with single param', () => {
@@ -137,7 +136,7 @@ describe('lexer', () => {
[TYPE.TAG, '/url', 17, 0, 16, 22],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('single fake tag', () => {
@@ -149,7 +148,7 @@ describe('lexer', () => {
[TYPE.WORD, 'user=111]', 2, 0, 2],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('single tag with spaces', () => {
@@ -160,7 +159,7 @@ describe('lexer', () => {
[TYPE.TAG, 'Single Tag', 0, 0, 0, 12],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
// @TODO: this is breaking change behavior
@@ -175,7 +174,7 @@ describe('lexer', () => {
[TYPE.TAG, '/textarea', 25, 0],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('tags with single word and camel case params', () => {
@@ -213,7 +212,7 @@ describe('lexer', () => {
[TYPE.SPACE, ' ', 28, 2, 203],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('string with quotemarks', () => {
@@ -232,7 +231,7 @@ describe('lexer', () => {
[TYPE.WORD, 'Adele', 22, 0],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('tags in brakets', () => {
@@ -249,7 +248,7 @@ describe('lexer', () => {
[TYPE.WORD, ']', 13, 0],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('tag as param', () => {
@@ -262,7 +261,7 @@ describe('lexer', () => {
[TYPE.TAG, '/color', 21, 0, 21, 29],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('tag with quotemark params with spaces', () => {
@@ -278,7 +277,7 @@ describe('lexer', () => {
[TYPE.TAG, '/url', 42, 0, 42, 48],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('tag with escaped quotemark param', () => {
@@ -292,7 +291,7 @@ describe('lexer', () => {
[TYPE.TAG, '/url', 26, 0, 26, 32],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('tag param without quotemarks', () => {
@@ -306,7 +305,7 @@ describe('lexer', () => {
[TYPE.TAG, '/style', 26, 0, 25, 33],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('list tag with items', () => {
@@ -344,7 +343,7 @@ describe('lexer', () => {
[TYPE.TAG, '/list', 0, 4, 52, 59],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('few tags without spaces', () => {
@@ -366,7 +365,7 @@ describe('lexer', () => {
[TYPE.TAG, '/mytag3', 74, 0, 74, 83],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('bad tags as texts', () => {
@@ -434,7 +433,7 @@ describe('lexer', () => {
const tokens = tokenize(input);
const output = asserts[idx];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
});
@@ -452,7 +451,7 @@ describe('lexer', () => {
[TYPE.TAG, 'Finger', 15, 0, 15, 23]
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('no close tag', () => {
@@ -467,7 +466,7 @@ describe('lexer', () => {
[TYPE.WORD, 'A', 13, 0],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('escaped tag', () => {
@@ -482,7 +481,7 @@ describe('lexer', () => {
[TYPE.WORD, '[', 9, 0],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('escaped tag and escaped backslash', () => {
@@ -502,7 +501,7 @@ describe('lexer', () => {
[TYPE.WORD, ']', 21, 0],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('context free tag [code]', () => {
@@ -520,12 +519,12 @@ describe('lexer', () => {
[TYPE.TAG, '/code', 25, 0, 25, 32],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('context free tag case insensitive [CODE]', () => {
const input = '[CODE] [b]some string[/b][/CODE]';
const tokens = tokenizeContextFreeTags(input, ['code']);
const tokens = tokenizeContextFreeTags('[CODE] [b]some string[/b][/CODE]', ['code']);
const output = [
[TYPE.TAG, 'CODE', 0, 0, 0, 6],
[TYPE.SPACE, ' ', 6, 0],
@@ -538,7 +537,7 @@ describe('lexer', () => {
[TYPE.TAG, '/CODE', 25, 0, 25, 32],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('bad closed tag with escaped backslash', () => {
@@ -552,7 +551,7 @@ describe('lexer', () => {
[TYPE.WORD, 'b]', 9, 0],
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
describe('html', () => {
@@ -575,7 +574,7 @@ describe('lexer', () => {
[TYPE.TAG, '/button', 78, 0, 78, 87]
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('attributes with no quotes or value', () => {
@@ -594,7 +593,7 @@ describe('lexer', () => {
[TYPE.TAG, '/button', 63, 0, 62, 71]
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test('attributes with no space between them. No valid, but accepted by the browser', () => {
@@ -612,7 +611,7 @@ describe('lexer', () => {
[TYPE.TAG, '/button', 76, 0, 76, 85]
];
expect(tokens).toBeMantchOutput(output);
expect(tokens).toBeMatchOutput(output);
});
test.skip('style tag', () => {
@@ -634,7 +633,7 @@ input.buttonred{cursor:hand;font-family:verdana;background:#d12124;color:#fff;he
-->
</style>`;
const tokens = tokenizeHTML(content);
expect(tokens).toBeMantchOutput([]);
expect(tokens).toBeMatchOutput([]);
});
test.skip('script tag', () => {
@@ -645,7 +644,7 @@ input.buttonred{cursor:hand;font-family:verdana;background:#d12124;color:#fff;he
//-->
</script>`;
const tokens = tokenizeHTML(content);
expect(tokens).toBeMantchOutput([]);
expect(tokens).toBeMatchOutput([]);
});
});
});
+153 -40
View File
@@ -1,12 +1,36 @@
import { parse } from '../src';
import type { TagNode, TagNodeTree } from "@bbob/types";
describe('Parser', () => {
const expectOutput = (ast: TagNodeTree, output: Partial<TagNodeTree>) => {
expect(ast).toBeInstanceOf(Array);
expect(ast).toMatchObject(output as {} | TagNode[]);
};
const astToJSON = (ast: TagNodeTree) => Array.isArray(ast) ? ast.map(item => {
if (typeof item === 'object' && typeof item.toJSON === 'function') {
return item.toJSON()
}
return item
}) : ast
declare global {
namespace jest {
interface Matchers<R> {
toBeMatchAST(expected: Array<unknown>): CustomMatcherResult;
}
}
}
expect.extend({
toBeMatchAST(ast, output) {
expect(astToJSON(ast)).toMatchObject(output as {} | TagNode[]);
return {
message: () =>
`no valid output`,
pass: true,
};
},
});
describe('Parser', () => {
test('parse paired tags tokens', () => {
const ast = parse('[best name=value]Foo Bar[/best]');
const output = [
@@ -31,7 +55,7 @@ describe('Parser', () => {
},
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
test('parse paired tags tokens 2', () => {
@@ -56,7 +80,7 @@ describe('Parser', () => {
},
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
describe('onlyAllowTags', () => {
@@ -87,7 +111,7 @@ describe('Parser', () => {
},
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
test('parse only allowed tags with params', () => {
@@ -96,7 +120,7 @@ describe('Parser', () => {
};
const ast = parse('hello [blah foo="bar"]world[/blah]', options);
expectOutput(ast, [
expect(ast).toBeMatchAST([
'hello',
' ',
'[blah foo="bar"]',
@@ -111,7 +135,7 @@ describe('Parser', () => {
};
const ast = parse('hello [blah="bar"]world[/blah]', options);
expectOutput(ast, [
expect(ast).toBeMatchAST([
'hello',
' ',
'[blah="bar"]',
@@ -180,7 +204,7 @@ describe('Parser', () => {
'[/tab]',
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
test('parse only allowed tags case insensitive', () => {
@@ -210,7 +234,7 @@ describe('Parser', () => {
},
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
});
@@ -243,8 +267,53 @@ describe('Parser', () => {
}
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
test('nesting similar context free tags [code][codeButton]text[/codeButton][/code]', () => {
const ast = parse('[code][codeButton]text[/codeButton][/code]', {
contextFreeTags: ['code']
});
const output = [
{
tag: 'code',
attrs: {},
content: [
'[',
'codeButton]text',
'[',
'/codeButton]'
]
}
];
expect(ast).toBeMatchAST(output);
})
test('broken nesting similar context free tags [code][codeButton]text[/codeButton][code]', () => {
const ast = parse('[code][codeButton]text[/codeButton][code]', {
contextFreeTags: ['code']
});
const output = [
{
attrs: {},
content: [],
tag: 'code',
},
{
attrs: {},
content: ['text'],
tag: 'codeButton',
},
{
attrs: {},
content: [],
tag: 'code',
},
];
expect(ast).toBeMatchAST(output);
})
});
describe('caseFreeTags', () => {
@@ -268,7 +337,7 @@ describe('Parser', () => {
"[/H1]"
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
test('case free tags', () => {
@@ -295,10 +364,57 @@ describe('Parser', () => {
}
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
})
test('nesting similar tags [code][codeButton]text[/codeButton][/code]', () => {
const ast = parse('[code][codeButton]text[/codeButton][/code]');
const output = [
{
tag: 'code',
attrs: {},
content: [
{
tag: 'codeButton',
attrs: {},
content: [
'text'
]
}
]
}
];
expect(ast).toBeMatchAST(output);
})
test('forgot close code tag [code][codeButton]text[/codeButton][code]', () => {
const ast = parse('[code][codeButton]text[/codeButton][code]');
const output = [
{
tag: 'code',
attrs: {},
content: []
},
{
tag: 'codeButton',
attrs: {},
content: [
'text'
]
},
{
tag: 'code',
attrs: {},
content: []
}
];
expect(ast).toBeMatchAST(output);
})
test('parse inconsistent tags', () => {
const ast = parse('[h1 name=value]Foo [Bar] /h1]');
const output = [
@@ -316,7 +432,7 @@ describe('Parser', () => {
'Foo',
' ',
{
tag: 'bar',
tag: 'Bar',
attrs: {},
content: [],
start: {
@@ -328,7 +444,7 @@ describe('Parser', () => {
'/h1]',
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
test('parse closed tag', () => {
@@ -337,7 +453,7 @@ describe('Parser', () => {
'[/h1]',
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
test('parse tag with value param', () => {
@@ -360,7 +476,7 @@ describe('Parser', () => {
},
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
test('parse tag with quoted param with spaces', () => {
@@ -385,7 +501,7 @@ describe('Parser', () => {
},
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
test('parse single tag with params', () => {
@@ -404,7 +520,7 @@ describe('Parser', () => {
},
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
test('detect inconsistent tag', () => {
@@ -463,14 +579,14 @@ describe('Parser', () => {
},
];
expectOutput(ast, output);
expect(ast).toBeMatchAST(output);
});
// @TODO: this is breaking change behavior
test.skip('parse tags with single attributes like disabled', () => {
const ast = parse('[b]hello[/b] [textarea disabled]world[/textarea]');
expectOutput(ast, [
expect(ast).toBeMatchAST([
{
tag: 'b',
attrs: {},
@@ -506,7 +622,7 @@ describe('Parser', () => {
test('parse url tag with get params', () => {
const ast = parse('[url=https://github.com/JiLiZART/bbob/search?q=any&unscoped_q=any]GET[/url]');
expectOutput(ast, [
expect(ast).toBeMatchAST([
{
tag: 'url',
attrs: {
@@ -531,7 +647,7 @@ describe('Parser', () => {
attr value"] this is a spoiler
[b]this is bold [i]this is bold and italic[/i] this is bold again[/b]
[/spoiler]this is outside again`);
expectOutput(ast, [
expect(ast).toBeMatchAST([
"this",
" ",
"is",
@@ -632,7 +748,7 @@ describe('Parser', () => {
[avatar href="/avatar/4/3/b/1606.jpg@20x20?cache=1561462725&bgclr=ffffff" size=xs][/avatar]
Group Name Go[/url] `);
expectOutput(ast, [
expect(ast).toBeMatchAST([
{
tag: 'url',
attrs: {
@@ -684,7 +800,7 @@ describe('Parser', () => {
test('parse url tag with # and = symbols [google docs]', () => {
const ast = parse('[url href=https://docs.google.com/spreadsheets/d/1W9VPUESF_NkbSa_HtRFrQNl0nYo8vPCxJFy7jD3Tpio/edit#gid=0]Docs[/url]');
expectOutput(ast, [
expect(ast).toBeMatchAST([
{
tag: 'url',
attrs: {
@@ -710,8 +826,7 @@ sdfasdfasdf
[url=xxx]xxx[/url]`;
expectOutput(
parse(str),
expect(parse(str)).toBeMatchAST(
[
{
tag: 'quote', attrs: {}, content: ['some'],
@@ -760,8 +875,7 @@ sdfasdfasdf
test('parse with lost closing tag on from', () => {
const str = `[quote]xxxsdfasdf[quote]some[/quote][color=red]test[/color]sdfasdfasdf[url=xxx]xxx[/url]`;
expectOutput(
parse(str),
expect(parse(str)).toBeMatchAST(
[
'[quote]',
'xxxsdfasdf',
@@ -806,8 +920,7 @@ sdfasdfasdf
test('parse with lost closing tag on to', () => {
const str = `[quote]some[/quote][color=red]test[/color]sdfasdfasdf[url=xxx]xxx[/url][quote]xxxsdfasdf`;
expectOutput(
parse(str),
expect(parse(str)).toBeMatchAST(
[
{
tag: 'quote', attrs: {}, content: ['some'],
@@ -852,7 +965,7 @@ sdfasdfasdf
test('parse with url in tag content', () => {
const input = parse('[img]https://tw.greywool.com/i/e3Ph5.png[/img]');
expectOutput(input, [
expect(input).toBeMatchAST([
{
tag: 'img',
attrs: {},
@@ -874,7 +987,7 @@ sdfasdfasdf
whitespaceInTags: false
})
expectOutput(input, [
expect(input).toBeMatchAST([
{
tag: 'b',
attrs: {},
@@ -913,7 +1026,7 @@ sdfasdfasdf
const content = `<button id="test0" class="value0" title="value1">class="value0" title="value1"</button>`;
const ast = parseHTML(content);
expectOutput(ast, [
expect(ast).toBeMatchAST([
{
"tag": "button",
"attrs": {
@@ -942,7 +1055,7 @@ sdfasdfasdf
const content = `<button id="test1" class=value2 disabled required>class=value2 disabled</button>`;
const ast = parseHTML(content);
expectOutput(ast, [
expect(ast).toBeMatchAST([
{
"tag": "button",
"attrs": {
@@ -972,7 +1085,7 @@ sdfasdfasdf
const content = `<button id="test2" class="value4"title="value5">class="value4"title="value5"</button>`;
const ast = parseHTML(content);
expectOutput(ast, [
expect(ast).toBeMatchAST([
{
"tag": "button",
"attrs": {
@@ -1000,7 +1113,7 @@ sdfasdfasdf
enableEscapeTags: true
});
expectOutput(ast, [
expect(ast).toBeMatchAST([
'[',
'b',
']',
@@ -1016,7 +1129,7 @@ sdfasdfasdf
enableEscapeTags: true
});
expectOutput(ast, [
expect(ast).toBeMatchAST([
'\\',
'[',
'b',