diff --git a/packages/bbob-parser/src/lexer.js b/packages/bbob-parser/src/lexer.js index c136094..34255ca 100644 --- a/packages/bbob-parser/src/lexer.js +++ b/packages/bbob-parser/src/lexer.js @@ -38,6 +38,7 @@ const createToken = (type, value, r = 0, cl = 0) => new Token(type, value, r, cl * @param {Function} options.onToken * @param {String} options.openTag * @param {String} options.closeTag + * @param {Boolean} options.enableEscapeTags * @return {Lexer} */ function createLexer(buffer, options = {}) { @@ -50,7 +51,10 @@ function createLexer(buffer, options = {}) { const closeTag = options.closeTag || CLOSE_BRAKET; const RESERVED_CHARS = [closeTag, openTag, QUOTEMARK, BACKSLASH, SPACE, TAB, EQ, N, EM]; - const NOT_CHAR_TOKENS = [openTag, SPACE, TAB, N]; + const NOT_CHAR_TOKENS = [ + ...(options.enableEscapeTags ? [BACKSLASH] : []), + openTag, SPACE, TAB, N, BACKSLASH, + ]; const WHITESPACES = [SPACE, TAB]; const SPECIAL_CHARS = [EQ, SPACE, TAB]; @@ -143,6 +147,7 @@ function createLexer(buffer, options = {}) { const next = () => { const currChar = bufferGrabber.getCurr(); + const nextChar = bufferGrabber.getNext(); if (currChar === N) { bufferGrabber.skip(); @@ -153,8 +158,12 @@ function createLexer(buffer, options = {}) { } else if (isWhiteSpace(currChar)) { const str = bufferGrabber.grabWhile(isWhiteSpace); emitToken(createToken(TYPE_SPACE, str, row, col)); + } else if (options.enableEscapeTags && currChar === BACKSLASH + && (nextChar === openTag || nextChar === closeTag)) { + bufferGrabber.skip(); // skip the \ without emitting anything + bufferGrabber.skip(); // skip past the [ or ] as well + emitToken(createToken(TYPE_WORD, nextChar, row, col)); } else if (currChar === openTag) { - const nextChar = bufferGrabber.getNext(); bufferGrabber.skip(); // skip openTag // detect case where we have '[My word [tag][/tag]' or we have '[My last line word' diff --git a/packages/bbob-parser/src/parse.js b/packages/bbob-parser/src/parse.js index dad94a5..1b7cd50 100644 --- a/packages/bbob-parser/src/parse.js +++ b/packages/bbob-parser/src/parse.js @@ -10,6 +10,7 @@ import { createList } from './utils'; * @param {Array} opts.onlyAllowTags * @param {String} opts.openTag * @param {String} opts.closeTag + * @param {Boolean} opts.enableEscapeTags * @return {Array} */ const parse = (input, opts = {}) => { @@ -220,6 +221,7 @@ const parse = (input, opts = {}) => { onlyAllowTags: options.onlyAllowTags, openTag: options.openTag, closeTag: options.closeTag, + enableEscapeTags: options.enableEscapeTags, }); // eslint-disable-next-line no-unused-vars diff --git a/packages/bbob-parser/test/lexer.test.js b/packages/bbob-parser/test/lexer.test.js index e73075a..1ee641d 100644 --- a/packages/bbob-parser/test/lexer.test.js +++ b/packages/bbob-parser/test/lexer.test.js @@ -288,6 +288,23 @@ describe('lexer', () => { expectOutput(output, tokens); }); + test('escaped tag', () => { + const tokenizeEscape = input => (createLexer(input, { + enableEscapeTags: true + }).tokenize()); + const input = '\\[b\\]test\\['; + const tokens = tokenizeEscape(input); + const output = [ + [TYPE.WORD, '[', '0', '0'], + [TYPE.WORD, 'b', '0', '0'], + [TYPE.WORD, ']', '0', '0'], + [TYPE.WORD, 'test', '0', '0'], + [TYPE.WORD, '[', '0', '0'], + ]; + + expectOutput(output, tokens); + }); + describe('html', () => { const tokenizeHTML = input => createLexer(input, { openTag: '<', closeTag: '>' }).tokenize(); diff --git a/packages/bbob-parser/test/parse.test.js b/packages/bbob-parser/test/parse.test.js index 53d2833..8ada554 100644 --- a/packages/bbob-parser/test/parse.test.js +++ b/packages/bbob-parser/test/parse.test.js @@ -183,5 +183,21 @@ describe('Parser', () => { } ]); }); + + test('parse escaped tags tags', () => { + const ast = parse('\\[b\\]test\\[/b\\]', { + enableEscapeTags: true + }); + + expectOutput(ast, [ + '[', + 'b', + ']', + 'test', + '[', + '/b', + ']', + ]); + }); }); });