mirrors/bbob
2
0
Fork 0
mirror of https://github.com/tenrok/BBob.git synced 2026-05-15 11:59:37 +03:00
Code Activity

feat: add start and end positions of tag nodes (#246)

Browse Source 
Closes #134

* feat: Add start and end positions of tag nodes

Improves accuracy of row/col error reporting. Now targets the start of the relevant token instead of the end.

* Simplify language for TagNode and Token

* Update static TagNode.create to ingest setStart() logic

improve readability of end pos offset for no attr tags
This commit is contained in:
Steven Chang
2024-08-01 00:42:29 -07:00
committed by GitHub
parent 0beab56d7f
commit 40848747d4
13 changed files with 929 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/ninety-points-explode.md
+17
View File
@@ -0,0 +1,17 @@
---
"@bbob/plugin-helper": minor
"@bbob/parser": minor
"@bbob/types": minor
"@bbob/cli": minor
"@bbob/core": minor
"@bbob/html": minor
"@bbob/preset": minor
"@bbob/preset-html5": minor
"@bbob/preset-react": minor
"@bbob/preset-vue": minor
"@bbob/react": minor
"@bbob/vue2": minor
"@bbob/vue3": minor
---
feat: Add start and end positions of tag nodes
packages/bbob-core/test/index.test.ts
+46 -22
View File
@@ -1,5 +1,5 @@
import { TagNode } from '@bbob/parser'
import core, { BBobPluginFunction, BBobPlugins } from '../src'
import { TagNode } from '@bbob/parser';
import core, { BBobPluginFunction, BBobPlugins } from '../src';
import { isTagNode } from "@bbob/plugin-helper";
const stringify = (val: unknown) => JSON.stringify(val);
@@ -11,15 +11,17 @@ describe('@bbob/core', () => {
const res = process([], '[style size="15px"]Large Text[/style]');
const ast = res.tree;
expect(res.html).toBe('[{"tag":"style","attrs":{"size":"15px"},"content":["Large"," ","Text"]}]');
expect(res.html).toBe('[{"tag":"style","attrs":{"size":"15px"},"content":["Large"," ","Text"],"start":{"from":0,"to":19},"end":{"from":29,"to":37}}]');
expect(ast).toBeInstanceOf(Array);
expect(stringify(ast)).toEqual(stringify([
{
tag: 'style',
attrs: { size: '15px' },
content: ["Large", " ", "Text"]
content: ["Large", " ", "Text"],
start: { from: 0, to: 19 },
end: { from: 29, to: 37 },
}
]))
]));
});
test('plugin walk api node', () => {
@@ -39,11 +41,11 @@ describe('@bbob/core', () => {
}
return node
return node;
});
return plugin
}
return plugin;
};
const res = process([testPlugin()], '[mytag size="15px"]Large Text[/mytag]');
const ast = res.tree;
@@ -61,7 +63,15 @@ describe('@bbob/core', () => {
' ',
'Text',
'Test'
]
],
start: {
from: 0,
to: 19
},
end: {
from: 29,
to: 37
}
}
]));
});
@@ -71,13 +81,13 @@ describe('@bbob/core', () => {
const plugin: BBobPluginFunction = (tree) => tree.walk(node => {
if (node === ':)') {
return TagNode.create('test-tag', {}, [])
return TagNode.create('test-tag', {}, []);
}
return node
})
return node;
});
return plugin
return plugin;
};
const res = process([testPlugin()], '[mytag]Large Text :)[/mytag]');
@@ -99,7 +109,15 @@ describe('@bbob/core', () => {
attrs: {},
content: [],
}
]
],
start: {
from: 0,
to: 7
},
end: {
from: 20,
to: 28
}
}
]));
});
@@ -109,13 +127,13 @@ describe('@bbob/core', () => {
const plugin: BBobPluginFunction = (tree) => tree.match([{ tag: 'mytag1' }, { tag: 'mytag2' }], node => {
if (isTagNode(node) && node.attrs) {
node.attrs['pass'] = 1
node.attrs['pass'] = 1;
}
return node
})
return node;
});
return plugin
return plugin;
};
const res = process([testPlugin()], `[mytag1 size="15"]Tag1[/mytag1][mytag2 size="16"]Tag2[/mytag2][mytag3]Tag3[/mytag3]`);
@@ -132,7 +150,9 @@ describe('@bbob/core', () => {
},
content: [
'Tag1'
]
],
start: { from: 0, to: 18 },
end: { from: 22, to: 31 }
},
{
tag: 'mytag2',
@@ -142,15 +162,19 @@ describe('@bbob/core', () => {
},
content: [
'Tag2'
]
],
start: { from: 31, to: 49 },
end: { from: 53, to: 62 }
},
{
tag: 'mytag3',
attrs: {},
content: [
'Tag3'
]
],
start: { from: 62, to: 70 },
end: { from: 74, to: 83 }
}
]));
})
});
});
packages/bbob-parser/src/Token.ts
+30 -10
View File
@@ -5,12 +5,14 @@ import {
} from '@bbob/plugin-helper';
import type { Token as TokenInterface } from "@bbob/types";
// type, value, line, row,
// type, value, line, row, start pos, end pos
const TOKEN_TYPE_ID = 't'; // 0;
const TOKEN_VALUE_ID = 'v'; // 1;
const TOKEN_COLUMN_ID = 'r'; // 2;
const TOKEN_LINE_ID = 'l'; // 3;
const TOKEN_START_POS_ID = 's'; // 4;
const TOKEN_END_POS_ID = 'e'; // 5;
const TOKEN_TYPE_WORD = 1; // 'word';
const TOKEN_TYPE_TAG = 2; // 'tag';
@@ -31,11 +33,15 @@ const getTokenLine = (token: Token) => (token && token[TOKEN_LINE_ID]) || 0;
const getTokenColumn = (token: Token) => (token && token[TOKEN_COLUMN_ID]) || 0;
const getStartPosition = (token: Token) => (token && token[TOKEN_START_POS_ID]) || 0;
const getEndPosition = (token: Token) => (token && token[TOKEN_END_POS_ID]) || 0;
const isTextToken = (token: Token) => {
if (token && typeof token[TOKEN_TYPE_ID] !== 'undefined') {
return token[TOKEN_TYPE_ID] === TOKEN_TYPE_SPACE
|| token[TOKEN_TYPE_ID] === TOKEN_TYPE_NEW_LINE
|| token[TOKEN_TYPE_ID] === TOKEN_TYPE_WORD;
|| token[TOKEN_TYPE_ID] === TOKEN_TYPE_NEW_LINE
|| token[TOKEN_TYPE_ID] === TOKEN_TYPE_WORD;
}
return false;
@@ -88,21 +94,25 @@ const tokenToText = (token: Token) => {
* @export
* @class Token
*/
class Token<TokenValue = string> implements TokenInterface {
readonly t: number // type
readonly v: string // value
readonly l: number // line
readonly r: number // row
class Token<TokenValue = string> implements TokenInterface {
readonly t: number; // type
readonly v: string; // value
readonly l: number; // line
readonly r: number; // row
readonly s: number; // start pos
readonly e: number; // end pos
constructor(type?: number, value?: TokenValue, row: number = 0, col: number = 0) {
constructor(type?: number, value?: TokenValue, row: number = 0, col: number = 0, start: number = 0, end: number = 0) {
this[TOKEN_LINE_ID] = row;
this[TOKEN_COLUMN_ID] = col;
this[TOKEN_TYPE_ID] = type || 0;
this[TOKEN_VALUE_ID] = String(value);
this[TOKEN_START_POS_ID] = start;
this[TOKEN_END_POS_ID] = end;
}
get type() {
return this[TOKEN_TYPE_ID]
return this[TOKEN_TYPE_ID];
}
isEmpty() {
@@ -149,6 +159,14 @@ class Token<TokenValue = string> implements TokenInterface {
return getTokenColumn(this);
}
getStart() {
return getStartPosition(this);
}
getEnd() {
return getEndPosition(this);
}
toString() {
return tokenToText(this);
}
@@ -158,6 +176,8 @@ export const TYPE_ID = TOKEN_TYPE_ID;
export const VALUE_ID = TOKEN_VALUE_ID;
export const LINE_ID = TOKEN_LINE_ID;
export const COLUMN_ID = TOKEN_COLUMN_ID;
export const START_POS_ID = TOKEN_START_POS_ID;
export const END_POS_ID = TOKEN_END_POS_ID;
export const TYPE_WORD = TOKEN_TYPE_WORD;
export const TYPE_TAG = TOKEN_TYPE_TAG;
export const TYPE_ATTR_NAME = TOKEN_TYPE_ATTR_NAME;
packages/bbob-parser/src/lexer.ts
+23 -10
View File
@@ -20,8 +20,8 @@ import { CharGrabber, createCharGrabber, trimChar, unquote } from './utils';
// for cases <!-- -->
const EM = '!';
export function createTokenOfType(type: number, value: string, r = 0, cl = 0) {
return new Token(type, value, r, cl)
export function createTokenOfType(type: number, value: string, r = 0, cl = 0, p = 0, e = 0) {
return new Token(type, value, r, cl, p, e);
}
const STATE_WORD = 0;
@@ -34,6 +34,7 @@ const TAG_STATE_VALUE = 2;
const WHITESPACES = [SPACE, TAB];
const SPECIAL_CHARS = [EQ, SPACE, TAB];
const END_POS_OFFSET = 2; // length + start position offset
const isWhiteSpace = (char: string) => (WHITESPACES.indexOf(char) >= 0);
const isEscapeChar = (char: string) => char === BACKSLASH;
@@ -43,6 +44,7 @@ const unq = (val: string) => unquote(trimChar(val, QUOTEMARK));
export function createLexer(buffer: string, options: LexerOptions = {}): LexerTokenizer {
let row = 0;
let prevCol = 0;
let col = 0;
let tokenIndex = -1;
@@ -89,16 +91,17 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
* @param {Number} type
* @param {String} value
*/
function emitToken(type: number, value: string) {
const token = createTokenOfType(type, value, row, col);
function emitToken(type: number, value: string, startPos?: number, endPos?: number) {
const token = createTokenOfType(type, value, row, prevCol, startPos, endPos);
onToken(token);
prevCol = col;
tokenIndex += 1;
tokens[tokenIndex] = token;
}
function nextTagState(tagChars: CharGrabber, isSingleValueTag: boolean) {
function nextTagState(tagChars: CharGrabber, isSingleValueTag: boolean, masterStartPos: number) {
if (tagMode === TAG_STATE_ATTR) {
const validAttrName = (char: string) => !(char === EQ || isWhiteSpace(char));
const name = tagChars.grabWhile(validAttrName);
@@ -161,6 +164,9 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
tagChars.skip();
emitToken(TYPE_ATTR_VALUE, unq(name));
if (tagChars.getPrev() === QUOTEMARK) {
prevCol++;
}
if (tagChars.isLast()) {
return TAG_STATE_NAME;
@@ -169,13 +175,15 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
return TAG_STATE_ATTR;
}
const start = masterStartPos + tagChars.getPos() - 1;
const validName = (char: string) => !(char === EQ || isWhiteSpace(char) || tagChars.isLast());
const name = tagChars.grabWhile(validName);
emitToken(TYPE_TAG, name);
emitToken(TYPE_TAG, name, start, masterStartPos + tagChars.getLength() + 1);
checkContextFreeMode(name);
tagChars.skip();
prevCol++;
// in cases when we has [url=someval]GET[/url] and we dont need to parse all
if (isSingleValueTag) {
@@ -209,11 +217,13 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
const isClosingTag = substr[0] === SLASH;
if (isNoAttrsInTag || isClosingTag) {
const startPos = chars.getPos() - 1;
const name = chars.grabWhile((char) => char !== closeTag);
const endPos = startPos + name.length + END_POS_OFFSET;
chars.skip(); // skip closeTag
emitToken(TYPE_TAG, name);
emitToken(TYPE_TAG, name, startPos, endPos);
checkContextFreeMode(name, isClosingTag);
return STATE_WORD;
@@ -223,6 +233,7 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
}
function stateAttrs() {
const startPos = chars.getPos();
const silent = true;
const tagStr = chars.grabWhile((char) => char !== closeTag, silent);
const tagGrabber = createCharGrabber(tagStr, { onSkip });
@@ -231,7 +242,7 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
tagMode = TAG_STATE_NAME;
while (tagGrabber.hasNext()) {
tagMode = nextTagState(tagGrabber, !hasSpace);
tagMode = nextTagState(tagGrabber, !hasSpace, startPos);
}
chars.skip(); // skip closeTag
@@ -246,6 +257,7 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
chars.skip();
col = 0;
prevCol = 0;
row++;
return STATE_WORD;
@@ -276,6 +288,7 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
emitToken(TYPE_WORD, chars.getCurr());
chars.skip();
prevCol++;
return STATE_WORD;
}
@@ -345,7 +358,7 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
if (nestedMap.has(value)) {
return !!nestedMap.get(value);
} else {
const status = (buffer.indexOf(value) > -1)
const status = (buffer.indexOf(value) > -1);
nestedMap.set(value, status);
@@ -356,5 +369,5 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
return {
tokenize,
isTokenNested,
}
};
}
packages/bbob-parser/src/parse.ts
+5 -1
View File
@@ -185,7 +185,7 @@ function parse(input: string, opts: ParseOptions = {}) {
function handleTagStart(token: Token) {
flushTagNodes();
const tagNode = TagNode.create(token.getValue(), {}, []);
const tagNode = TagNode.create(token.getValue(), {}, [], { from: token.getStart(), to: token.getEnd() });
const isNested = isTokenNested(token);
tagNodes.push(tagNode);
@@ -203,6 +203,10 @@ function parse(input: string, opts: ParseOptions = {}) {
* @param {Token} token
*/
function handleTagEnd(token: Token) {
const lastTagNode = nestedNodes.last();
if (isTagNode(lastTagNode)) {
lastTagNode.setEnd({ from: token.getStart(), to: token.getEnd() });
}
flushTagNodes();
const lastNestedNode = nestedNodes.flush();
packages/bbob-parser/src/utils.ts
+8
View File
@@ -42,6 +42,14 @@ export class CharGrabber {
return this.s[this.c.pos]
}
getPos() {
return this.c.pos;
}
getLength() {
return this.c.len;
}
getRest() {
return this.s.substring(this.c.pos)
}
packages/bbob-parser/test/Token.test.ts
+12 -2
View File
@@ -1,10 +1,10 @@
import Token, { TYPE_WORD, TYPE_TAG, TYPE_ATTR_NAME, TYPE_ATTR_VALUE, TYPE_SPACE, TYPE_NEW_LINE } from '../src/Token'
import Token, { TYPE_WORD, TYPE_TAG, TYPE_ATTR_NAME, TYPE_ATTR_VALUE, TYPE_SPACE, TYPE_NEW_LINE } from '../src/Token';
describe('Token', () => {
test('isEmpty', () => {
const token = new Token();
expect(token.isEmpty()).toBeTruthy()
expect(token.isEmpty()).toBeTruthy();
});
test('isText', () => {
const token = new Token(TYPE_WORD);
@@ -56,6 +56,16 @@ describe('Token', () => {
expect(token.getColumn()).toBe(14);
});
test('getStartPos', () => {
const token = new Token(TYPE_TAG, 'my-tag', 12, 14, 50);
expect(token.getStart()).toBe(50);
});
test('getEndPos', () => {
const token = new Token(TYPE_TAG, 'my-tag', 12, 14, 50, 60);
expect(token.getEnd()).toBe(60);
});
test('toString', () => {
const tokenEnd = new Token(TYPE_TAG, '/my-tag', 12, 14);
packages/bbob-parser/test/lexer.test.ts
+276 -245
View File
@@ -1,5 +1,5 @@
import { TYPE_ID, VALUE_ID, TYPE_WORD, TYPE_TAG, TYPE_ATTR_NAME, TYPE_ATTR_VALUE, TYPE_SPACE, TYPE_NEW_LINE} from '../src/Token'
import { createLexer } from '../src/lexer'
import { TYPE_ID, VALUE_ID, TYPE_WORD, TYPE_TAG, TYPE_ATTR_NAME, TYPE_ATTR_VALUE, TYPE_SPACE, TYPE_NEW_LINE, LINE_ID, COLUMN_ID, START_POS_ID, END_POS_ID } from '../src/Token';
import { createLexer } from '../src/lexer';
declare global {
namespace jest {
@@ -32,19 +32,19 @@ describe('lexer', () => {
if (tokens.length !== output.length) {
return {
message: () =>
`expected tokens length ${tokens.length} to be ${output.length}`,
`expected tokens length ${tokens.length} to be ${output.length}`,
pass: false,
};
}
for (let idx = 0; idx < tokens.length; idx++) {
const token = tokens[idx];
const [type, value] = output[idx];
const [type, value, col, row, startPos, endPos] = output[idx];
if (typeof token !== 'object') {
return {
message: () =>
`token must to be Object`,
`token must to be Object`,
pass: false,
};
}
@@ -52,7 +52,7 @@ describe('lexer', () => {
if (token[TYPE_ID] !== type) {
return {
message: () =>
`expected token type ${TYPE_NAMES[type]} but recieved ${TYPE_NAMES[token[TYPE_ID]]} for ${JSON.stringify(output[idx])}`,
`expected token type ${TYPE_NAMES[type]} but received ${TYPE_NAMES[token[TYPE_ID]]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
@@ -60,7 +60,39 @@ describe('lexer', () => {
if (token[VALUE_ID] !== value) {
return {
message: () =>
`expected token value ${value} but recieved ${token[VALUE_ID]} for ${JSON.stringify(output[idx])}`,
`expected token value ${value} but received ${token[VALUE_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (token[LINE_ID] !== row) {
return {
message: () =>
`expected token row ${row} but received ${token[LINE_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (token[COLUMN_ID] !== col) {
return {
message: () =>
`expected token col ${col} but received ${token[COLUMN_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (type === TYPE.TAG && token[START_POS_ID] !== startPos) {
return {
message: () =>
`expected token start pos ${startPos} but received ${token[START_POS_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
if (type === TYPE.TAG && token[END_POS_ID] !== endPos) {
return {
message: () =>
`expected token end pos ${endPos} but received ${token[END_POS_ID]} for ${JSON.stringify(output[idx])}`,
pass: false,
};
}
@@ -68,7 +100,7 @@ describe('lexer', () => {
return {
message: () =>
`no valid output`,
`no valid output`,
pass: true,
};
},
@@ -78,7 +110,7 @@ describe('lexer', () => {
const input = '[SingleTag]';
const tokens = tokenize(input);
const output = [
[TYPE.TAG, 'SingleTag', '0', '0'],
[TYPE.TAG, 'SingleTag', 0, 0, 0, 11],
];
expect(tokens).toBeMantchOutput(output);
@@ -88,8 +120,8 @@ describe('lexer', () => {
const input = '[user=111]';
const tokens = tokenize(input);
const output = [
[TYPE.TAG, 'user', '0', '0'],
[TYPE.ATTR_VALUE, '111', '0', '0'],
[TYPE.TAG, 'user', 0, 0, 0, 10],
[TYPE.ATTR_VALUE, '111', 6, 0],
];
expect(tokens).toBeMantchOutput(output);
@@ -99,10 +131,10 @@ describe('lexer', () => {
const input = '[url=someval]GET[/url]';
const tokens = tokenize(input);
const output = [
[TYPE.TAG, 'url', '0', '0'],
[TYPE.ATTR_VALUE, 'someval', '0', '0'],
[TYPE.WORD, 'GET', '0', '0'],
[TYPE.TAG, '/url', '0', '0'],
[TYPE.TAG, 'url', 0, 0, 0, 13],
[TYPE.ATTR_VALUE, 'someval', 5, 0],
[TYPE.WORD, 'GET', 13, 0],
[TYPE.TAG, '/url', 17, 0, 16, 22],
];
expect(tokens).toBeMantchOutput(output);
@@ -112,9 +144,9 @@ describe('lexer', () => {
const input = '[ user=111]';
const tokens = tokenize(input);
const output = [
[TYPE.WORD, '[', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.WORD, 'user=111]', '0', '0'],
[TYPE.WORD, '[', 0, 0, 0],
[TYPE.SPACE, ' ', 1, 0, 1],
[TYPE.WORD, 'user=111]', 2, 0, 2],
];
expect(tokens).toBeMantchOutput(output);
@@ -125,7 +157,7 @@ describe('lexer', () => {
const tokens = tokenize(input);
const output = [
[TYPE.TAG, 'Single Tag', '0', '0'],
[TYPE.TAG, 'Single Tag', 0, 0, 0, 12],
];
expect(tokens).toBeMantchOutput(output);
@@ -137,10 +169,10 @@ describe('lexer', () => {
const tokens = tokenize(input);
const output = [
[TYPE.TAG, 'textarea', '0', '0'],
[TYPE.ATTR_VALUE, 'disabled', '0', '0'],
[TYPE.WORD, 'world"', '0', '0'],
[TYPE.TAG, '/textarea', '0', '0'],
[TYPE.TAG, 'textarea', 0, 0],
[TYPE.ATTR_VALUE, 'disabled', 10, 0],
[TYPE.WORD, 'world"', 19, 0],
[TYPE.TAG, '/textarea', 25, 0],
];
expect(tokens).toBeMantchOutput(output);
@@ -153,32 +185,32 @@ describe('lexer', () => {
const tokens = tokenize(input);
const output = [
[TYPE.TAG, 'url', '0', '0'],
[TYPE.ATTR_NAME, 'href', '0', '0'],
[TYPE.ATTR_VALUE, '/groups/123/', '0', '0'],
[TYPE.ATTR_NAME, 'isNowrap', '0', '0'],
[TYPE.ATTR_VALUE, 'true', '0', '0'],
[TYPE.ATTR_NAME, 'isTextOverflow', '0', '0'],
[TYPE.ATTR_VALUE, 'true', '0', '0'],
[TYPE.ATTR_NAME, 'state', '0', '0'],
[TYPE.ATTR_VALUE, 'primary', '0', '0'],
[TYPE.NEW_LINE, '\n', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.TAG, 'avatar', '0', '0'],
[TYPE.ATTR_NAME, 'href', '0', '0'],
[TYPE.ATTR_VALUE, '/avatar/4/3/b/1606.jpg@20x20?cache=1561462725&bgclr=ffffff', '0', '0'],
[TYPE.ATTR_NAME, 'size', '0', '0'],
[TYPE.ATTR_VALUE, 'xs', '0', '0'],
[TYPE.TAG, '/avatar', '0', '0'],
[TYPE.NEW_LINE, '\n', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.WORD, 'Group', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.WORD, 'Name', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.WORD, 'Go', '0', '0'],
[TYPE.TAG, '/url', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.TAG, 'url', 0, 0, 0, 73],
[TYPE.ATTR_NAME, 'href', 5, 0, 5],
[TYPE.ATTR_VALUE, '/groups/123/', 10, 0, 10],
[TYPE.ATTR_NAME, 'isNowrap', 25, 0, 25],
[TYPE.ATTR_VALUE, 'true', 34, 0, 34],
[TYPE.ATTR_NAME, 'isTextOverflow', 39, 0, 39],
[TYPE.ATTR_VALUE, 'true', 54, 0, 54],
[TYPE.ATTR_NAME, 'state', 59, 0, 59],
[TYPE.ATTR_VALUE, 'primary', 65, 0, 65],
[TYPE.NEW_LINE, '\n', 73, 0, 73],
[TYPE.SPACE, ' ', 0, 1, 74],
[TYPE.TAG, 'avatar', 8, 1, 82, 164],
[TYPE.ATTR_NAME, 'href', 16, 1, 90],
[TYPE.ATTR_VALUE, '/avatar/4/3/b/1606.jpg@20x20?cache=1561462725&bgclr=ffffff', 21, 1, 95],
[TYPE.ATTR_NAME, 'size', 82, 1, 156],
[TYPE.ATTR_VALUE, 'xs', 87, 1, 161],
[TYPE.TAG, '/avatar', 90, 1, 164, 173],
[TYPE.NEW_LINE, '\n', 100, 1, 173],
[TYPE.SPACE, ' ', 0, 2, 174],
[TYPE.WORD, 'Group', 9, 2, 184],
[TYPE.SPACE, ' ', 14, 2, 189],
[TYPE.WORD, 'Name', 15, 2, 190],
[TYPE.SPACE, ' ', 19, 2, 194],
[TYPE.WORD, 'Go', 20, 2, 195],
[TYPE.TAG, '/url', 22, 2, 196, 202],
[TYPE.SPACE, ' ', 28, 2, 203],
];
expect(tokens).toBeMantchOutput(output);
@@ -189,15 +221,15 @@ describe('lexer', () => {
const tokens = tokenize(input);
const output = [
[TYPE.WORD, '"Someone', '0', '0'],
[TYPE.SPACE, ' ', '8', '0'],
[TYPE.WORD, 'Like', '8', '0'],
[TYPE.SPACE, ' ', '13', '0'],
[TYPE.WORD, 'You"', '13', '0'],
[TYPE.SPACE, ' ', '18', '0'],
[TYPE.WORD, 'by', '18', '0'],
[TYPE.SPACE, ' ', '21', '0'],
[TYPE.WORD, 'Adele', '21', '0'],
[TYPE.WORD, '"Someone', 0, 0],
[TYPE.SPACE, ' ', 8, 0],
[TYPE.WORD, 'Like', 9, 0],
[TYPE.SPACE, ' ', 13, 0],
[TYPE.WORD, 'You"', 14, 0],
[TYPE.SPACE, ' ', 18, 0],
[TYPE.WORD, 'by', 19, 0],
[TYPE.SPACE, ' ', 21, 0],
[TYPE.WORD, 'Adele', 22, 0],
];
expect(tokens).toBeMantchOutput(output);
@@ -208,13 +240,13 @@ describe('lexer', () => {
const tokens = tokenize(input);
const output = [
[TYPE.WORD, '[', '0', '0'],
[TYPE.SPACE, ' ', '1', '0'],
[TYPE.TAG, 'h1', '2', '0'],
[TYPE.WORD, 'G', '1', '0'],
[TYPE.TAG, '/h1', '7', '0'],
[TYPE.SPACE, ' ', '12', '0'],
[TYPE.WORD, ']', '7', '0'],
[TYPE.WORD, '[', 0, 0],
[TYPE.SPACE, ' ', 1, 0],
[TYPE.TAG, 'h1', 2, 0, 2, 6],
[TYPE.WORD, 'G', 6, 0],
[TYPE.TAG, '/h1', 7, 0, 7, 12],
[TYPE.SPACE, ' ', 12, 0],
[TYPE.WORD, ']', 13, 0],
];
expect(tokens).toBeMantchOutput(output);
@@ -224,10 +256,10 @@ describe('lexer', () => {
const input = '[color="#ff0000"]Text[/color]';
const tokens = tokenize(input);
const output = [
[TYPE.TAG, 'color', '0', '0'],
[TYPE.ATTR_VALUE, '#ff0000', '6', '0'],
[TYPE.WORD, 'Text', '17', '0'],
[TYPE.TAG, '/color', '21', '0'],
[TYPE.TAG, 'color', 0, 0, 0, 17],
[TYPE.ATTR_VALUE, '#ff0000', 7, 0],
[TYPE.WORD, 'Text', 17, 0],
[TYPE.TAG, '/color', 21, 0, 21, 29],
];
expect(tokens).toBeMantchOutput(output);
@@ -237,13 +269,13 @@ describe('lexer', () => {
const input = '[url text="Foo Bar" text2="Foo Bar 2"]Text[/url]';
const tokens = tokenize(input);
const output = [
[TYPE.TAG, 'url', '0', '0'],
[TYPE.ATTR_NAME, 'text', '4', '0'],
[TYPE.ATTR_VALUE, 'Foo Bar', '9', '0'],
[TYPE.ATTR_NAME, 'text2', '4', '0'],
[TYPE.ATTR_VALUE, 'Foo Bar 2', '9', '0'],
[TYPE.WORD, 'Text', '20', '0'],
[TYPE.TAG, '/url', '24', '0'],
[TYPE.TAG, 'url', 0, 0, 0, 38],
[TYPE.ATTR_NAME, 'text', 5, 0],
[TYPE.ATTR_VALUE, 'Foo Bar', 10, 0],
[TYPE.ATTR_NAME, 'text2', 20, 0],
[TYPE.ATTR_VALUE, 'Foo Bar 2', 26, 0],
[TYPE.WORD, 'Text', 38, 0],
[TYPE.TAG, '/url', 42, 0, 42, 48],
];
expect(tokens).toBeMantchOutput(output);
@@ -253,11 +285,11 @@ describe('lexer', () => {
const input = `[url text="Foo \\"Bar"]Text[/url]`;
const tokens = tokenize(input);
const output = [
[TYPE.TAG, 'url', '0', '0'],
[TYPE.ATTR_NAME, 'text', '4', '0'],
[TYPE.ATTR_VALUE, 'Foo "Bar', '9', '0'],
[TYPE.WORD, 'Text', '22', '0'],
[TYPE.TAG, '/url', '26', '0'],
[TYPE.TAG, 'url', 0, 0, 0, 22],
[TYPE.ATTR_NAME, 'text', 5, 0],
[TYPE.ATTR_VALUE, 'Foo "Bar', 10, 0],
[TYPE.WORD, 'Text', 22, 0],
[TYPE.TAG, '/url', 26, 0, 26, 32],
];
expect(tokens).toBeMantchOutput(output);
@@ -267,11 +299,11 @@ describe('lexer', () => {
const input = '[style color=#ff0000]Text[/style]';
const tokens = tokenize(input);
const output = [
[TYPE.TAG, 'style', '0', '0'],
[TYPE.ATTR_NAME, 'color', '6', '0'],
[TYPE.ATTR_VALUE, '#ff0000', '12', '0'],
[TYPE.WORD, 'Text', '21', '0'],
[TYPE.TAG, '/style', '25', '0'],
[TYPE.TAG, 'style', 0, 0, 0, 21],
[TYPE.ATTR_NAME, 'color', 7, 0],
[TYPE.ATTR_VALUE, '#ff0000', 13, 0],
[TYPE.WORD, 'Text', 21, 0],
[TYPE.TAG, '/style', 26, 0, 25, 33],
];
expect(tokens).toBeMantchOutput(output);
@@ -286,30 +318,30 @@ describe('lexer', () => {
const tokens = tokenize(input);
const output = [
[TYPE.TAG, 'list', '0', '0'],
[TYPE.NEW_LINE, '\n', '6', '0'],
[TYPE.SPACE, ' ', '0', '1'],
[TYPE.TAG, '*', '3', '1'],
[TYPE.SPACE, ' ', '6', '1'],
[TYPE.WORD, 'Item', '7', '1'],
[TYPE.SPACE, ' ', '11', '1'],
[TYPE.WORD, '1.', '11', '1'],
[TYPE.NEW_LINE, '\n', '14', '1'],
[TYPE.SPACE, ' ', '0', '2'],
[TYPE.TAG, '*', '3', '2'],
[TYPE.SPACE, ' ', '6', '2'],
[TYPE.WORD, 'Item', '14', '1'],
[TYPE.SPACE, ' ', '11', '2'],
[TYPE.WORD, '2.', '11', '2'],
[TYPE.NEW_LINE, '\n', '14', '2'],
[TYPE.SPACE, ' ', '0', '3'],
[TYPE.TAG, '*', '3', '3'],
[TYPE.SPACE, ' ', '6', '3'],
[TYPE.WORD, 'Item', '14', '2'],
[TYPE.SPACE, ' ', '11', '3'],
[TYPE.WORD, '3.', '11', '3'],
[TYPE.NEW_LINE, '\n', '14', '3'],
[TYPE.TAG, '/list', '0', '4'],
[TYPE.TAG, 'list', 0, 0, 0, 6],
[TYPE.NEW_LINE, '\n', 6, 0],
[TYPE.SPACE, ' ', 0, 1],
[TYPE.TAG, '*', 3, 1, 10, 13],
[TYPE.SPACE, ' ', 6, 1],
[TYPE.WORD, 'Item', 7, 1],
[TYPE.SPACE, ' ', 11, 1],
[TYPE.WORD, '1.', 12, 1],
[TYPE.NEW_LINE, '\n', 14, 1],
[TYPE.SPACE, ' ', 0, 2],
[TYPE.TAG, '*', 3, 2, 25, 28],
[TYPE.SPACE, ' ', 6, 2],
[TYPE.WORD, 'Item', 7, 2],
[TYPE.SPACE, ' ', 11, 2],
[TYPE.WORD, '2.', 12, 2],
[TYPE.NEW_LINE, '\n', 14, 2],
[TYPE.SPACE, ' ', 0, 3],
[TYPE.TAG, '*', 3, 3, 40, 43],
[TYPE.SPACE, ' ', 6, 3],
[TYPE.WORD, 'Item', 7, 3],
[TYPE.SPACE, ' ', 11, 3],
[TYPE.WORD, '3.', 12, 3],
[TYPE.NEW_LINE, '\n', 14, 3],
[TYPE.TAG, '/list', 0, 4, 52, 59],
];
expect(tokens).toBeMantchOutput(output);
@@ -319,19 +351,19 @@ describe('lexer', () => {
const input = '[mytag1 size="15"]Tag1[/mytag1][mytag2 size="16"]Tag2[/mytag2][mytag3]Tag3[/mytag3]';
const tokens = tokenize(input);
const output = [
[TYPE.TAG, 'mytag1', 0, 0],
[TYPE.ATTR_NAME, 'size', 0, 0],
[TYPE.ATTR_VALUE, '15', 0, 0],
[TYPE.WORD, 'Tag1', 0, 0],
[TYPE.TAG, '/mytag1', 0, 0],
[TYPE.TAG, 'mytag2', 0, 0],
[TYPE.ATTR_NAME, 'size', 0, 0],
[TYPE.ATTR_VALUE, '16', 0, 0],
[TYPE.WORD, 'Tag2', 0, 0],
[TYPE.TAG, '/mytag2', 0, 0],
[TYPE.TAG, 'mytag3', 0, 0],
[TYPE.WORD, 'Tag3', 0, 0],
[TYPE.TAG, '/mytag3', 0, 0],
[TYPE.TAG, 'mytag1', 0, 0, 0, 18],
[TYPE.ATTR_NAME, 'size', 8, 0],
[TYPE.ATTR_VALUE, '15', 13, 0],
[TYPE.WORD, 'Tag1', 18, 0],
[TYPE.TAG, '/mytag1', 22, 0, 22, 31],
[TYPE.TAG, 'mytag2', 31, 0, 31, 49],
[TYPE.ATTR_NAME, 'size', 39, 0],
[TYPE.ATTR_VALUE, '16', 44, 0],
[TYPE.WORD, 'Tag2', 49, 0],
[TYPE.TAG, '/mytag2', 53, 0, 53, 62],
[TYPE.TAG, 'mytag3', 62, 0, 62, 70],
[TYPE.WORD, 'Tag3', 70, 0],
[TYPE.TAG, '/mytag3', 74, 0, 74, 83],
];
expect(tokens).toBeMantchOutput(output);
@@ -351,51 +383,50 @@ describe('lexer', () => {
const asserts = [
[
[TYPE.WORD, '[', '0', '0'],
[TYPE.WORD, ']', '0', '0']
[TYPE.WORD, '[', 0, 0],
[TYPE.WORD, ']', 1, 0]
],
[
[TYPE.WORD, '[', '0', '0'],
[TYPE.WORD, '=]', '0', '0']
[TYPE.WORD, '[', 0, 0],
[TYPE.WORD, '=]', 1, 0]
],
[
[TYPE.WORD, '!', '0', '0'],
[TYPE.WORD, '[', '1', '0'],
[TYPE.WORD, '](image.jpg)', '1', '0'],
// [TYPE.WORD, '', '1', '0'],
[TYPE.WORD, '!', 0, 0],
[TYPE.WORD, '[', 1, 0],
[TYPE.WORD, '](image.jpg)', 2, 0],
],
[
[TYPE.WORD, 'x', '0', '0'],
[TYPE.SPACE, ' ', '1', '0'],
[TYPE.WORD, 'html(', '1', '0'],
[TYPE.TAG, 'a. title', '7', '0'],
[TYPE.TAG, ', alt', '17', '0'],
[TYPE.TAG, ', classes', '24', '0'],
[TYPE.WORD, ')', '7', '0'],
[TYPE.SPACE, ' ', '36', '0'],
[TYPE.WORD, 'x', '36', '0'],
[TYPE.WORD, 'x', 0, 0],
[TYPE.SPACE, ' ', 1, 0],
[TYPE.WORD, 'html(', 2, 0],
[TYPE.TAG, 'a. title', 7, 0, 7, 17],
[TYPE.TAG, ', alt', 17, 0, 17, 24],
[TYPE.TAG, ', classes', 24, 0, 24, 35],
[TYPE.WORD, ')', 35, 0],
[TYPE.SPACE, ' ', 36, 0],
[TYPE.WORD, 'x', 37, 0],
],
[
[TYPE.TAG, '/y', '0', '0']
[TYPE.TAG, '/y', 0, 0, 0, 4]
],
[
[TYPE.WORD, '[', '0', '0'],
[TYPE.WORD, 'sc', '0', '0']
[TYPE.WORD, '[', 0, 0],
[TYPE.WORD, 'sc', 1, 0]
],
[
// [sc /
[TYPE.WORD, '[', '0', '0'],
[TYPE.WORD, 'sc', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.WORD, '/', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.TAG, '/sc', '0', '0']
[TYPE.WORD, '[', 0, 0],
[TYPE.WORD, 'sc', 1, 0],
[TYPE.SPACE, ' ', 3, 0],
[TYPE.WORD, '/', 4, 0],
[TYPE.SPACE, ' ', 5, 0],
[TYPE.TAG, '/sc', 6, 0, 6, 11]
],
[
[TYPE.WORD, '[', '0', '0'],
[TYPE.WORD, 'sc', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.WORD, 'arg="val', '0', '0'],
[TYPE.WORD, '[', 0, 0],
[TYPE.WORD, 'sc', 1, 0],
[TYPE.SPACE, ' ', 3, 0],
[TYPE.WORD, 'arg="val', 4, 0],
]
];
@@ -411,14 +442,14 @@ describe('lexer', () => {
const input = `[Finger Part A [Finger]`;
const tokens = tokenize(input);
const output = [
[TYPE.WORD, '[', '0', '0'],
[TYPE.WORD, 'Finger', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.WORD, 'Part', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.WORD, 'A', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.TAG, 'Finger', '0', '0']
[TYPE.WORD, '[', 0, 0],
[TYPE.WORD, 'Finger', 1, 0],
[TYPE.SPACE, ' ', 7, 0],
[TYPE.WORD, 'Part', 8, 0],
[TYPE.SPACE, ' ', 12, 0],
[TYPE.WORD, 'A', 13, 0],
[TYPE.SPACE, ' ', 14, 0],
[TYPE.TAG, 'Finger', 15, 0, 15, 23]
];
expect(tokens).toBeMantchOutput(output);
@@ -428,12 +459,12 @@ describe('lexer', () => {
const input = '[Finger Part A';
const tokens = tokenize(input);
const output = [
[TYPE.WORD, '[', '0', '0'],
[TYPE.WORD, 'Finger', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.WORD, 'Part', '0', '0'],
[TYPE.SPACE, ' ', '0', '0'],
[TYPE.WORD, 'A', '0', '0'],
[TYPE.WORD, '[', 0, 0],
[TYPE.WORD, 'Finger', 1, 0],
[TYPE.SPACE, ' ', 7, 0],
[TYPE.WORD, 'Part', 8, 0],
[TYPE.SPACE, ' ', 12, 0],
[TYPE.WORD, 'A', 13, 0],
];
expect(tokens).toBeMantchOutput(output);
@@ -444,11 +475,11 @@ describe('lexer', () => {
const tokens = tokenizeEscape(input);
const output = [
[TYPE.WORD, '[', '0', '0'],
[TYPE.WORD, 'b', '0', '0'],
[TYPE.WORD, ']', '0', '0'],
[TYPE.WORD, 'test', '0', '0'],
[TYPE.WORD, '[', '0', '0'],
[TYPE.WORD, '[', 0, 0],
[TYPE.WORD, 'b', 2, 0],
[TYPE.WORD, ']', 3, 0],
[TYPE.WORD, 'test', 5, 0],
[TYPE.WORD, '[', 9, 0],
];
expect(tokens).toBeMantchOutput(output);
@@ -458,67 +489,67 @@ describe('lexer', () => {
const input = '\\\\\\[b\\\\\\]test\\\\\\[/b\\\\\\]';
const tokens = tokenizeEscape(input);
const output = [
[TYPE.WORD, '\\', '0', '0'],
[TYPE.WORD, '[', '0', '0'],
[TYPE.WORD, 'b', '0', '0'],
[TYPE.WORD, '\\', '0', '0'],
[TYPE.WORD, ']', '0', '0'],
[TYPE.WORD, 'test', '0', '0'],
[TYPE.WORD, '\\', '0', '0'],
[TYPE.WORD, '[', '0', '0'],
[TYPE.WORD, '/b', '0', '0'],
[TYPE.WORD, '\\', '0', '0'],
[TYPE.WORD, ']', '0', '0'],
[TYPE.WORD, '\\', 0, 0],
[TYPE.WORD, '[', 2, 0],
[TYPE.WORD, 'b', 4, 0],
[TYPE.WORD, '\\', 5, 0],
[TYPE.WORD, ']', 7, 0],
[TYPE.WORD, 'test', 9, 0],
[TYPE.WORD, '\\', 13, 0],
[TYPE.WORD, '[', 15, 0],
[TYPE.WORD, '/b', 17, 0],
[TYPE.WORD, '\\', 19, 0],
[TYPE.WORD, ']', 21, 0],
];
expect(tokens).toBeMantchOutput(output);
});
test('context free tag [code]', () => {
const input = '[code] [b]some string[/b][/code]'
const input = '[code] [b]some string[/b][/code]';
const tokens = tokenizeContextFreeTags(input, ['code']);
const output = [
[TYPE.TAG, 'code', 0, 0],
[TYPE.SPACE, ' ', 0, 0],
[TYPE.WORD, '[', 0, 0],
[TYPE.WORD, 'b]some', 0, 0],
[TYPE.SPACE, ' ', 0, 0],
[TYPE.WORD, 'string', 0, 0],
[TYPE.WORD, '[', 0, 0],
[TYPE.WORD, '/b]', 0, 0],
[TYPE.TAG, '/code', 0, 0],
]
[TYPE.TAG, 'code', 0, 0, 0, 6],
[TYPE.SPACE, ' ', 6, 0],
[TYPE.WORD, '[', 7, 0],
[TYPE.WORD, 'b]some', 8, 0],
[TYPE.SPACE, ' ', 14, 0],
[TYPE.WORD, 'string', 15, 0],
[TYPE.WORD, '[', 21, 0],
[TYPE.WORD, '/b]', 22, 0],
[TYPE.TAG, '/code', 25, 0, 25, 32],
];
expect(tokens).toBeMantchOutput(output);
})
});
test('context free tag case insensitive [CODE]', () => {
const input = '[CODE] [b]some string[/b][/CODE]'
const input = '[CODE] [b]some string[/b][/CODE]';
const tokens = tokenizeContextFreeTags(input, ['code']);
const output = [
[TYPE.TAG, 'CODE', 0, 0],
[TYPE.SPACE, ' ', 0, 0],
[TYPE.WORD, '[', 0, 0],
[TYPE.WORD, 'b]some', 0, 0],
[TYPE.SPACE, ' ', 0, 0],
[TYPE.WORD, 'string', 0, 0],
[TYPE.WORD, '[', 0, 0],
[TYPE.WORD, '/b]', 0, 0],
[TYPE.TAG, '/CODE', 0, 0],
]
[TYPE.TAG, 'CODE', 0, 0, 0, 6],
[TYPE.SPACE, ' ', 6, 0],
[TYPE.WORD, '[', 7, 0],
[TYPE.WORD, 'b]some', 8, 0],
[TYPE.SPACE, ' ', 14, 0],
[TYPE.WORD, 'string', 15, 0],
[TYPE.WORD, '[', 21, 0],
[TYPE.WORD, '/b]', 22, 0],
[TYPE.TAG, '/CODE', 25, 0, 25, 32],
];
expect(tokens).toBeMantchOutput(output);
})
});
test('bad closed tag with escaped backslash', () => {
const input = `[b]test[\\b]`;
const tokens = tokenizeEscape(input);
const output = [
[TYPE.TAG, 'b', '0', '3'],
[TYPE.WORD, 'test', '0', '7'],
[TYPE.WORD, '[', '0', '8'],
[TYPE.WORD, '\\', '0', '9'],
[TYPE.WORD, 'b]', '0', '11'],
[TYPE.TAG, 'b', 0, 0, 0, 3],
[TYPE.WORD, 'test', 3, 0],
[TYPE.WORD, '[', 7, 0],
[TYPE.WORD, '\\', 8, 0],
[TYPE.WORD, 'b]', 9, 0],
];
expect(tokens).toBeMantchOutput(output);
@@ -531,17 +562,17 @@ describe('lexer', () => {
const content = `<button id="test0" class="value0" title="value1">class="value0" title="value1"</button>`;
const tokens = tokenizeHTML(content);
const output = [
[TYPE.TAG, 'button', 2, 0],
[TYPE.ATTR_NAME, 'id', 2, 0],
[TYPE.ATTR_VALUE, 'test0', 2, 0],
[TYPE.ATTR_NAME, 'class', 2, 0],
[TYPE.ATTR_VALUE, 'value0', 2, 0],
[TYPE.ATTR_NAME, 'title', 2, 0],
[TYPE.ATTR_VALUE, 'value1', 2, 0],
[TYPE.WORD, "class=\"value0\"", 2, 0],
[TYPE.SPACE, " ", 2, 0],
[TYPE.WORD, "title=\"value1\"", 2, 0],
[TYPE.TAG, '/button', 2, 0]
[TYPE.TAG, 'button', 0, 0, 0, 49],
[TYPE.ATTR_NAME, 'id', 8, 0],
[TYPE.ATTR_VALUE, 'test0', 11, 0],
[TYPE.ATTR_NAME, 'class', 19, 0],
[TYPE.ATTR_VALUE, 'value0', 25, 0],
[TYPE.ATTR_NAME, 'title', 34, 0],
[TYPE.ATTR_VALUE, 'value1', 40, 0],
[TYPE.WORD, "class=\"value0\"", 49, 0],
[TYPE.SPACE, " ", 63, 0],
[TYPE.WORD, "title=\"value1\"", 64, 0],
[TYPE.TAG, '/button', 78, 0, 78, 87]
];
expect(tokens).toBeMantchOutput(output);
@@ -551,16 +582,16 @@ describe('lexer', () => {
const content = `<button id="test1" class=value2 disabled>class=value2 disabled</button>`;
const tokens = tokenizeHTML(content);
const output = [
[TYPE.TAG, 'button', 2, 0],
[TYPE.ATTR_NAME, 'id', 2, 0],
[TYPE.ATTR_VALUE, 'test1', 2, 0],
[TYPE.ATTR_NAME, 'class', 2, 0],
[TYPE.ATTR_VALUE, 'value2', 2, 0],
[TYPE.ATTR_VALUE, 'disabled', 2, 0],
[TYPE.WORD, "class=value2", 2, 0],
[TYPE.SPACE, " ", 2, 0],
[TYPE.WORD, "disabled", 2, 0],
[TYPE.TAG, '/button', 2, 0]
[TYPE.TAG, 'button', 0, 0, 0, 41],
[TYPE.ATTR_NAME, 'id', 8, 0],
[TYPE.ATTR_VALUE, 'test1', 11, 0],
[TYPE.ATTR_NAME, 'class', 19, 0],
[TYPE.ATTR_VALUE, 'value2', 25, 0],
[TYPE.ATTR_VALUE, 'disabled', 32, 0],
[TYPE.WORD, "class=value2", 41, 0],
[TYPE.SPACE, " ", 54, 0],
[TYPE.WORD, "disabled", 55, 0],
[TYPE.TAG, '/button', 63, 0, 62, 71]
];
expect(tokens).toBeMantchOutput(output);
@@ -570,15 +601,15 @@ describe('lexer', () => {
const content = `<button id="test2" class="value4"title="value5">class="value4"title="value5"</button>`;
const tokens = tokenizeHTML(content);
const output = [
[TYPE.TAG, 'button', 2, 0],
[TYPE.ATTR_NAME, 'id', 2, 0],
[TYPE.ATTR_VALUE, 'test2', 2, 0],
[TYPE.ATTR_NAME, 'class', 2, 0],
[TYPE.ATTR_VALUE, 'value4', 2, 0],
[TYPE.ATTR_NAME, 'title', 2, 0],
[TYPE.ATTR_VALUE, 'value5', 2, 0],
[TYPE.WORD, "class=\"value4\"title=\"value5\"", 2, 0],
[TYPE.TAG, '/button', 2, 0]
[TYPE.TAG, 'button', 0, 0, 0, 48],
[TYPE.ATTR_NAME, 'id', 8, 0],
[TYPE.ATTR_VALUE, 'test2', 11, 0],
[TYPE.ATTR_NAME, 'class', 19, 0],
[TYPE.ATTR_VALUE, 'value4', 25, 0],
[TYPE.ATTR_NAME, 'title', 34, 0],
[TYPE.ATTR_VALUE, 'value5', 39, 0],
[TYPE.WORD, "class=\"value4\"title=\"value5\"", 48, 0],
[TYPE.TAG, '/button', 76, 0, 76, 85]
];
expect(tokens).toBeMantchOutput(output);
@@ -601,7 +632,7 @@ input{padding:0px;margin:0px;font-size:9pt}
input.medium{width:100px;height:18px}
input.buttonred{cursor:hand;font-family:verdana;background:#d12124;color:#fff;height:1.4em;font-weight:bold;font-size:9pt;padding:0px 2px;margin:0px;border:0px none #000}
-->
</style>`
</style>`;
const tokens = tokenizeHTML(content);
expect(tokens).toBeMantchOutput([]);
});
@@ -615,6 +646,6 @@ input.buttonred{cursor:hand;font-family:verdana;background:#d12124;color:#fff;he
</script>`;
const tokens = tokenizeHTML(content);
expect(tokens).toBeMantchOutput([]);
})
})
});
});
});
packages/bbob-parser/test/parse.test.ts
+437 -65
View File
@@ -1,10 +1,10 @@
import { parse } from '../src'
import type { TagNodeTree } from "@bbob/plugin-helper";
import { parse } from '../src';
import type { TagNode, TagNodeTree } from "@bbob/types";
describe('Parser', () => {
const expectOutput = (ast: TagNodeTree, output: Partial<TagNodeTree>) => {
expect(ast).toBeInstanceOf(Array);
expect(ast).toEqual(output);
expect(ast).toMatchObject(output as {} | TagNode[]);
};
test('parse paired tags tokens', () => {
@@ -20,6 +20,14 @@ describe('Parser', () => {
' ',
'Bar',
],
start: {
from: 0,
to: 17,
},
end: {
from: 24,
to: 31,
},
},
];
@@ -37,6 +45,14 @@ describe('Parser', () => {
' ',
'Bar',
],
start: {
from: 0,
to: 5,
},
end: {
from: 12,
to: 18,
},
},
];
@@ -60,6 +76,14 @@ describe('Parser', () => {
'[Bar]',
' '
],
start: {
from: 0,
to: 15,
},
end: {
from: 25,
to: 30,
},
},
];
@@ -78,7 +102,7 @@ describe('Parser', () => {
'[blah foo="bar"]',
'world',
'[/blah]'
])
]);
});
test('parse only allowed tags with named param', () => {
@@ -93,7 +117,7 @@ describe('Parser', () => {
'[blah="bar"]',
'world',
'[/blah]'
])
]);
});
test('parse only allowed tags inside disabled tags', () => {
@@ -107,6 +131,14 @@ describe('Parser', () => {
tag: 'ch',
attrs: {},
content: ['E'],
start: {
from: 7,
to: 11,
},
end: {
from: 12,
to: 17,
},
},
'\n',
'A',
@@ -126,6 +158,14 @@ describe('Parser', () => {
tag: 'ch',
attrs: {},
content: ['A'],
start: {
from: 81,
to: 85,
},
end: {
from: 86,
to: 91,
},
},
'\n',
'All',
@@ -159,12 +199,20 @@ describe('Parser', () => {
'[Bar]',
' '
],
start: {
from: 0,
to: 15,
},
end: {
from: 25,
to: 30,
},
},
];
expectOutput(ast, output);
});
})
});
describe('contextFreeTags', () => {
test('context free tag [code]', () => {
@@ -176,20 +224,28 @@ describe('Parser', () => {
tag: 'code',
attrs: {},
content: [
' ',
'[',
'b]some',
' ',
'string',
'[',
'/b]'
]
' ',
'[',
'b]some',
' ',
'string',
'[',
'/b]'
],
start: {
from: 0,
to: 6,
},
end: {
from: 25,
to: 32,
},
}
]
];
expectOutput(ast, output);
})
})
});
});
test('parse inconsistent tags', () => {
const ast = parse('[h1 name=value]Foo [Bar] /h1]');
@@ -199,14 +255,22 @@ describe('Parser', () => {
name: 'value'
},
tag: 'h1',
content: []
content: [],
start: {
from: 0,
to: 15,
},
},
'Foo',
' ',
{
tag: 'bar',
attrs: {},
content: []
content: [],
start: {
from: 19,
to: 24,
},
},
' ',
'/h1]',
@@ -224,6 +288,14 @@ describe('Parser', () => {
'https://github.com/jilizart/bbob': 'https://github.com/jilizart/bbob',
},
content: ['BBob'],
start: {
from: 0,
to: 38,
},
end: {
from: 42,
to: 48,
},
},
];
@@ -241,6 +313,14 @@ describe('Parser', () => {
text: 'Foo Bar',
},
content: ['Text'],
start: {
from: 0,
to: 64,
},
end: {
from: 68,
to: 74,
},
},
];
@@ -256,6 +336,10 @@ describe('Parser', () => {
'https://github.com/jilizart/bbob': 'https://github.com/jilizart/bbob',
},
content: [],
start: {
from: 0,
to: 38,
},
},
];
@@ -279,6 +363,14 @@ describe('Parser', () => {
size: '15',
},
content: ['Tag1'],
start: {
from: 0,
to: 18,
},
end: {
from: 22,
to: 31,
},
},
{
tag: 'mytag2',
@@ -286,11 +378,27 @@ describe('Parser', () => {
size: '16',
},
content: ['Tag2'],
start: {
from: 31,
to: 49,
},
end: {
from: 53,
to: 62,
},
},
{
tag: 'mytag3',
attrs: {},
content: ['Tag3'],
start: {
from: 62,
to: 70,
},
end: {
from: 74,
to: 83,
},
},
];
@@ -306,6 +414,14 @@ describe('Parser', () => {
tag: 'b',
attrs: {},
content: ['hello'],
start: {
from: 0,
to: 17,
},
end: {
from: 24,
to: 31,
},
},
' ',
{
@@ -314,6 +430,14 @@ describe('Parser', () => {
disabled: 'disabled',
},
content: ['world'],
start: {
from: 0,
to: 17,
},
end: {
from: 24,
to: 31,
},
},
]);
});
@@ -328,10 +452,120 @@ describe('Parser', () => {
'https://github.com/JiLiZART/bbob/search?q=any&unscoped_q=any': 'https://github.com/JiLiZART/bbob/search?q=any&unscoped_q=any',
},
content: ['GET'],
start: {
from: 0,
to: 66,
},
end: {
from: 69,
to: 75,
},
},
]);
});
test('parse triple nested tags', () => {
const ast = parse(`this is outside [spoiler title="name with
multiline
attr value"] this is a spoiler
[b]this is bold [i]this is bold and italic[/i] this is bold again[/b]
[/spoiler]this is outside again`);
expectOutput(ast, [
"this",
" ",
"is",
" ",
"outside",
" ",
{
attrs: {
"title": "name with\n multiline\n attr value",
},
content: [
" ",
"this",
" ",
"is",
" ",
"a",
" ",
"spoiler",
"\n",
" ",
{
attrs: {},
content: [
"this",
" ",
"is",
" ",
"bold",
" ",
{
attrs: {},
content: [
"this",
" ",
"is",
" ",
"bold",
" ",
"and",
" ",
"italic",
],
end: {
to: 147,
from: 143,
},
start: {
"to": 120,
"from": 117,
},
"tag": "i",
},
" ",
"this",
" ",
"is",
" ",
"bold",
" ",
"again",
],
end: {
"to": 170,
"from": 166,
},
start: {
"to": 104,
"from": 101,
},
tag: "b",
},
"\n",
" ",
],
end: {
"to": 187,
"from": 177,
},
start: {
"to": 76,
"from": 16,
},
tag: "spoiler",
},
"this",
" ",
"is",
" ",
"outside",
" ",
"again",
]);
});
test('parse tag with camelCase params', () => {
const ast = parse(`[url href="/groups/123/" isNowrap=true isTextOverflow=true state=primary]
[avatar href="/avatar/4/3/b/1606.jpg@20x20?cache=1561462725&bgclr=ffffff" size=xs][/avatar]
@@ -355,7 +589,15 @@ describe('Parser', () => {
href: '/avatar/4/3/b/1606.jpg@20x20?cache=1561462725&bgclr=ffffff',
size: 'xs'
},
content: []
content: [],
start: {
from: 82,
to: 164,
},
end: {
from: 164,
to: 173,
},
},
'\n',
' ',
@@ -365,6 +607,14 @@ describe('Parser', () => {
' ',
'Go',
],
start: {
from: 0,
to: 73,
},
end: {
from: 196,
to: 202,
},
},
' ',
]);
@@ -380,6 +630,14 @@ describe('Parser', () => {
href: 'https://docs.google.com/spreadsheets/d/1W9VPUESF_NkbSa_HtRFrQNl0nYo8vPCxJFy7jD3Tpio/edit#gid=0',
},
content: ['Docs'],
start: {
from: 0,
to: 105,
},
end: {
from: 109,
to: 115,
},
},
]);
});
@@ -389,56 +647,146 @@ describe('Parser', () => {
[quote]xxxsdfasdf
sdfasdfasdf
[url=xxx]xxx[/url]`
[url=xxx]xxx[/url]`;
expectOutput(
parse(str),
[
{ tag: 'quote', attrs: {}, content: ['some'] },
{ tag: 'color', attrs: { red: 'red' }, content: ['test'] },
'\n',
'[quote]',
'xxxsdfasdf',
'\n',
'sdfasdfasdf',
'\n',
'\n',
{ tag: 'url', attrs: { xxx: 'xxx' }, content: ['xxx'] }
]
)
})
parse(str),
[
{
tag: 'quote', attrs: {}, content: ['some'],
start: {
from: 0,
to: 7,
},
end: {
from: 11,
to: 19,
},
},
{
tag: 'color', attrs: { red: 'red' }, content: ['test'],
start: {
from: 19,
to: 30,
},
end: {
from: 34,
to: 42,
},
},
'\n',
'[quote]',
'xxxsdfasdf',
'\n',
'sdfasdfasdf',
'\n',
'\n',
{
tag: 'url', attrs: { xxx: 'xxx' }, content: ['xxx'],
start: {
from: 74,
to: 83,
},
end: {
from: 86,
to: 92,
},
}
]
);
});
test('parse with lost closing tag on start', () => {
const str = `[quote]xxxsdfasdf[quote]some[/quote][color=red]test[/color]sdfasdfasdf[url=xxx]xxx[/url]`
test('parse with lost closing tag on from', () => {
const str = `[quote]xxxsdfasdf[quote]some[/quote][color=red]test[/color]sdfasdfasdf[url=xxx]xxx[/url]`;
expectOutput(
parse(str),
[
'[quote]',
'xxxsdfasdf',
{ tag: 'quote', attrs: {}, content: ['some'] },
{ tag: 'color', attrs: { red: 'red' }, content: ['test'] },
'sdfasdfasdf',
{ tag: 'url', attrs: { xxx: 'xxx' }, content: ['xxx'] }
]
)
})
parse(str),
[
'[quote]',
'xxxsdfasdf',
{
tag: 'quote', attrs: {}, content: ['some'],
start: {
from: 17,
to: 24,
},
end: {
from: 28,
to: 36,
},
},
{
tag: 'color', attrs: { red: 'red' }, content: ['test'],
start: {
from: 36,
to: 47,
},
end: {
from: 51,
to: 59,
},
},
'sdfasdfasdf',
{
tag: 'url', attrs: { xxx: 'xxx' }, content: ['xxx'],
start: {
from: 70,
to: 79,
},
end: {
from: 82,
to: 88,
},
}
]
);
});
test('parse with lost closing tag on end', () => {
const str = `[quote]some[/quote][color=red]test[/color]sdfasdfasdf[url=xxx]xxx[/url][quote]xxxsdfasdf`
test('parse with lost closing tag on to', () => {
const str = `[quote]some[/quote][color=red]test[/color]sdfasdfasdf[url=xxx]xxx[/url][quote]xxxsdfasdf`;
expectOutput(
parse(str),
[
{ tag: 'quote', attrs: {}, content: ['some'] },
{ tag: 'color', attrs: { red: 'red' }, content: ['test'] },
'sdfasdfasdf',
{ tag: 'url', attrs: { xxx: 'xxx' }, content: ['xxx'] },
'[quote]',
'xxxsdfasdf',
]
)
})
parse(str),
[
{
tag: 'quote', attrs: {}, content: ['some'],
start: {
from: 0,
to: 7,
},
end: {
from: 11,
to: 19,
},
},
{
tag: 'color', attrs: { red: 'red' }, content: ['test'],
start: {
from: 19,
to: 30,
},
end: {
from: 34,
to: 42,
},
},
'sdfasdfasdf',
{
tag: 'url', attrs: { xxx: 'xxx' }, content: ['xxx'],
start: {
from: 53,
to: 62,
},
end: {
from: 65,
to: 71,
},
},
'[quote]',
'xxxsdfasdf',
]
);
});
describe('html', () => {
const parseHTML = (input: string) => parse(input, { openTag: '<', closeTag: '>' });
@@ -459,7 +807,15 @@ sdfasdfasdf
"class=\"value0\"",
" ",
"title=\"value1\""
]
],
start: {
from: 0,
to: 49,
},
end: {
from: 78,
to: 87,
},
}
]);
});
@@ -481,7 +837,15 @@ sdfasdfasdf
"class=value2",
" ",
"disabled"
]
],
start: {
from: 0,
to: 50,
},
end: {
from: 71,
to: 80,
},
}
]);
});
@@ -500,7 +864,15 @@ sdfasdfasdf
},
"content": [
"class=\"value4\"title=\"value5\""
]
],
start: {
from: 0,
to: 48,
},
end: {
from: 76,
to: 85,
},
}
]);
});
packages/bbob-plugin-helper/src/TagNode.ts
+41 -20
View File
@@ -1,4 +1,4 @@
import type { NodeContent, TagNodeObject, TagNodeTree } from "@bbob/types";
import type { NodeContent, TagNodeObject, TagNodeTree, TagPosition } from "@bbob/types";
import { OPEN_BRAKET, CLOSE_BRAKET, SLASH } from './char';
import {
@@ -30,38 +30,40 @@ const getTagAttrs = <AttrValue>(tag: string, params: Record<string, AttrValue>)
const renderContent = (content: TagNodeTree, openTag: string, closeTag: string) => {
const toString = (node: NodeContent) => {
if (isTagNode(node)) {
return node.toString({ openTag, closeTag })
return node.toString({ openTag, closeTag });
}
return String(node)
}
return String(node);
};
if (Array.isArray(content)) {
return content.reduce<string>((r, node) => {
if (node !== null) {
return r + toString(node)
return r + toString(node);
}
return r
}, '')
return r;
}, '');
}
if (content) {
return toString(content)
return toString(content);
}
return null
}
return null;
};
export class TagNode<TagValue extends any = any> implements TagNodeObject {
public readonly tag: string | TagValue
public attrs: Record<string, unknown>
public content: TagNodeTree
public readonly tag: string | TagValue;
public attrs: Record<string, unknown>;
public content: TagNodeTree;
public start?: TagPosition;
public end?: TagPosition;
constructor(tag: string | TagValue, attrs: Record<string, unknown>, content: TagNodeTree) {
this.tag = tag;
this.attrs = attrs;
this.content = content
this.content = content;
}
attr(name: string, value?: unknown) {
@@ -76,6 +78,14 @@ export class TagNode<TagValue extends any = any> implements TagNodeObject {
return appendToNode(this, value);
}
setStart(value: TagPosition) {
this.start = value;
}
setEnd(value: TagPosition) {
this.end = value;
}
get length(): number {
return getNodeLength(this);
}
@@ -91,25 +101,36 @@ export class TagNode<TagValue extends any = any> implements TagNodeObject {
}
toTagNode() {
return new TagNode(String(this.tag).toLowerCase(), this.attrs, this.content);
const newNode = new TagNode(String(this.tag).toLowerCase(), this.attrs, this.content);
if (this.start) {
newNode.setStart(this.start);
}
if (this.end) {
newNode.setEnd(this.end);
}
return newNode;
}
toString({ openTag = OPEN_BRAKET, closeTag = CLOSE_BRAKET } = {}): string {
const content = this.content ? renderContent(this.content, openTag, closeTag) : ''
const content = this.content ? renderContent(this.content, openTag, closeTag) : '';
const tagStart = this.toTagStart({ openTag, closeTag });
if (this.content === null || Array.isArray(this.content) && this.content.length === 0) {
if (this.content === null || Array.isArray(this.content) && this.content.length === 0) {
return tagStart;
}
return `${tagStart}${content}${this.toTagEnd({ openTag, closeTag })}`;
}
static create(tag: string, attrs: Record<string, unknown> = {}, content: TagNodeTree = null) {
return new TagNode(tag, attrs, content)
static create(tag: string, attrs: Record<string, unknown> = {}, content: TagNodeTree = null, start?: TagPosition) {
const node = new TagNode(tag, attrs, content);
if (start) {
node.setStart(start);
}
return node;
}
static isOf(node: TagNode, type: string) {
return (node.tag === type)
return (node.tag === type);
}
}
packages/bbob-plugin-helper/test/TagNode.test.ts
+19 -2
View File
@@ -2,7 +2,7 @@ import { TagNode } from '../src'
describe('@bbob/plugin-helper/TagNode', () => {
test('create', () => {
const tagNode = TagNode.create('test', {test: 1}, ['Hello']);
const tagNode = TagNode.create('test', {test: 1}, ['Hello'], {from: 0, to: 10});
expect(tagNode).toBeInstanceOf(TagNode)
});
@@ -36,12 +36,15 @@ describe('@bbob/plugin-helper/TagNode', () => {
});
test('toTagNode', () => {
const tagNode = TagNode.create('test', {test: 1}, ['Hello']);
const tagNode = TagNode.create('test', {test: 1}, ['Hello'], {from: 0, to: 10});
tagNode.setEnd({from: 20, to: 27});
const newTagNode = tagNode.toTagNode()
expect(newTagNode !== tagNode).toBe(true);
expect(newTagNode.tag).toEqual(tagNode.tag);
expect(newTagNode.content).toEqual(tagNode.content);
expect(newTagNode.start).toEqual(tagNode.start);
expect(newTagNode.end).toEqual(tagNode.end);
});
test('null content', () => {
@@ -56,6 +59,20 @@ describe('@bbob/plugin-helper/TagNode', () => {
expect(String(tagNode)).toBe('[img]');
});
test('setStart', () => {
const tagNode = TagNode.create('test', {test: 1}, ['Hello']);
tagNode.setStart({from: 0, to: 10});
expect(tagNode.start).toEqual({from: 0, to: 10});
});
test('setEnd', () => {
const tagNode = TagNode.create('test', {test: 1}, ['Hello']);
tagNode.setEnd({from: 20, to: 27});
expect(tagNode.end).toEqual({from: 20, to: 27});
});
describe('toString', () => {
test('tag with content and params', () => {
const tagNode = TagNode.create('test', {test: 1}, ['Hello']);
packages/bbob-types/src/common.ts
+11 -7
View File
@@ -1,13 +1,17 @@
export type StringNode = string | number
export type StringNode = string | number;
export interface TagNodeObject<TagValue extends any = any> {
readonly tag: TagValue
attrs?: Record<string, unknown>
content?: TagNodeTree<TagValue>
readonly tag: TagValue;
attrs?: Record<string, unknown>;
content?: TagNodeTree<TagValue>;
start?: TagPosition;
end?: TagPosition;
}
export type NodeContent<TagValue extends any = any> = TagNodeObject<TagValue> | StringNode | null
export type NodeContent<TagValue extends any = any> = TagNodeObject<TagValue> | StringNode | null;
export type PartialNodeContent<TagValue extends any = any> = Partial<TagNodeObject<TagValue>> | StringNode | null
export type PartialNodeContent<TagValue extends any = any> = Partial<TagNodeObject<TagValue>> | StringNode | null;
export type TagNodeTree<TagValue extends any = any> = NodeContent<TagValue> | NodeContent<TagValue>[] | null
export type TagNodeTree<TagValue extends any = any> = NodeContent<TagValue> | NodeContent<TagValue>[] | null;
export type TagPosition = { from: number; to: number; };
packages/bbob-types/src/parser.ts
+4 -2
View File
@@ -1,4 +1,4 @@
import { TagNodeTree } from "./common";
import { TagNodeTree, TagPosition } from "./common";
export interface ParseError {
tagName: string;
@@ -9,7 +9,9 @@ export interface ParseError {
export interface TagNode {
readonly tag: string
attrs?: Record<string, unknown>
content?: TagNodeTree
content?: TagNodeTree,
start?: TagPosition;
end?: TagPosition;
}
export interface Token<TokenValue = string> {