mirrors/bbob
2
0
Fork 0
mirror of https://github.com/tenrok/BBob.git synced 2026-06-17 19:21:20 +03:00
Code Activity

fix(parser): tokenizer error with quotemark strings

Browse Source
This commit is contained in:
Nikolay Kostyurin
2018-07-11 23:37:51 +02:00
parent 3119314b07
commit 7f400506d4
6 changed files with 64 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/bbob-parser/test/Tokenizer.test.js
+19
View File
@@ -34,6 +34,25 @@ describe('Tokenizer', () => {
expectOutput(output, tokens);
});
test('tokenize string with quotemarks', () => {
const input = '"Someone Like You" by Adele';
const tokens = tokenize(input);
const output = [
[TYPE.WORD, '"Someone', '0', '0'],
[TYPE.SPACE, ' ', '8', '0'],
[TYPE.WORD, 'Like', '8', '0'],
[TYPE.SPACE, ' ', '13', '0'],
[TYPE.WORD, 'You"', '13', '0'],
[TYPE.SPACE, ' ', '18', '0'],
[TYPE.WORD, 'by', '18', '0'],
[TYPE.SPACE, ' ', '21', '0'],
[TYPE.WORD, 'Adele', '21', '0'],
];
expectOutput(output, tokens);
});
test('tokenize tag as param', () => {
const input = '[color="#ff0000"]Text[/color]';
const tokens = tokenize(input);