mirrors/bbob
2
0
Fork 0
mirror of https://github.com/tenrok/BBob.git synced 2026-06-11 18:02:26 +03:00
Code Activity

add Tokenizer tests

Browse Source
This commit is contained in:
Nikolay Kostyurin
2018-06-09 00:06:36 +02:00
parent d739ec8ffd
commit 5e34dd9d43
9 changed files with 232 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/bbob-parser/Tokenizer.test.js
+85 -4
View File
@@ -1,14 +1,95 @@
const Tokenizer = require('./Tokenizer');
const TYPE = Tokenizer.TYPE;
describe("Tokenizer", () => {
it("tokenize single tag", () => {
test("tokenize single tag", () => {
const input = `[SingleTag]`;
const tokens = new Tokenizer(input).tokenize();
expect(tokens).toBeInstanceOf(Array);
expect(tokens).toEqual([
[TYPE.TAG, 'SingleTag', 0, 0]
])
});
test("tokenize single tag with spaces", () => {
const input = `[Single Tag]`;
const tokens = new Tokenizer(input).tokenize();
expect(tokens).toBeInstanceOf(Array);
expect(tokens).toEqual([
[TYPE.TAG, 'Single Tag', 0, 0]
])
});
test("tokenize tag as param", () => {
const input = `[color="#ff0000"]Text[/color]`;
const tokens = new Tokenizer(input).tokenize();
expect(tokens).toBeInstanceOf(Array);
expect(tokens).toEqual([
[TYPE.TAG, 'color', 0, 0],
[TYPE.ATTR_VALUE, '#ff0000', 6, 0],
[TYPE.WORD, 'Text', 17, 0],
[TYPE.TAG, '/color', 21, 0]
])
});
test("tokenize tag param without quotemarks", () => {
const input = `[style color=#ff0000]Text[/style]`;
const tokens = new Tokenizer(input).tokenize();
expect(tokens).toBeInstanceOf(Array);
expect(tokens).toEqual([
[TYPE.TAG, 'style', 0, 0],
[TYPE.ATTR_NAME, 'color', 6, 0],
[TYPE.ATTR_VALUE, '#ff0000', 12, 0],
[TYPE.WORD, 'Text', 21, 0],
[TYPE.TAG, '/style', 25, 0]
])
});
test("tokenize list tag with items", () => {
const input = `[list]
[*] Item 1.
[*] Item 2.
[*] Item 3.
[/list]`;
const tokens = new Tokenizer(input).tokenize();
console.log('tokens', tokens);
expect(tokens).toBeInstanceOf(Array);
expect(tokens[0]).toEqual(['tag', 'SingleTag', 0, 0])
expect(tokens).toEqual([
[TYPE.TAG, 'list', 0, 0],
[TYPE.NEW_LINE, '\n', 6, 0],
[TYPE.SPACE, ' ', 0, 1],
[TYPE.SPACE, ' ', 1, 1],
[TYPE.SPACE, ' ', 2, 1],
[TYPE.TAG, '*', 3, 1],
[TYPE.SPACE, ' ', 6, 1],
[TYPE.WORD, 'Item', 7, 1],
[TYPE.SPACE, ' ', 11, 1],
[TYPE.WORD, '1.', 11, 1],
[TYPE.NEW_LINE, '\n', 14, 1],
[TYPE.SPACE, ' ', 0, 2],
[TYPE.SPACE, ' ', 1, 2],
[TYPE.SPACE, ' ', 2, 2],
[TYPE.TAG, '*', 3, 2],
[TYPE.SPACE, ' ', 6, 2],
[TYPE.WORD, 'Item', 14, 1],
[TYPE.SPACE, ' ', 11, 2],
[TYPE.WORD, '2.', 11, 2],
[TYPE.NEW_LINE, '\n', 14, 2],
[TYPE.SPACE, ' ', 0, 3],
[TYPE.SPACE, ' ', 1, 3],
[TYPE.SPACE, ' ', 2, 3],
[TYPE.TAG, '*', 3, 3],
[TYPE.SPACE, ' ', 6, 3],
[TYPE.WORD, 'Item', 14, 2],
[TYPE.SPACE, ' ', 11, 3],
[TYPE.WORD, '3.', 11, 3],
[TYPE.NEW_LINE, '\n', 14, 3],
[TYPE.TAG, '/list', 0, 4]
])
})
});