feat(parse): allow tags to be escaped with backslash (#17)

* feat(parse): allow tags to be escaped with backslash

adds additional option `enableEscapeTags` to `parse` and `createLexer` that
when true will parse openTag and closeTag as WORD (rather than TAG) when
proceeded with backslash

packages/bbob-parser/src/lexer.js

@@ -38,6 +38,7 @@ const createToken = (type, value, r = 0, cl = 0) => new Token(type, value, r, cl
  * @param {Function} options.onToken
  * @param {String} options.openTag
  * @param {String} options.closeTag
+ * @param {Boolean} options.enableEscapeTags
  * @return {Lexer}
  */
 function createLexer(buffer, options = {}) {
@@ -50,7 +51,10 @@ function createLexer(buffer, options = {}) {
   const closeTag = options.closeTag || CLOSE_BRAKET;
 
   const RESERVED_CHARS = [closeTag, openTag, QUOTEMARK, BACKSLASH, SPACE, TAB, EQ, N, EM];
-  const NOT_CHAR_TOKENS = [openTag, SPACE, TAB, N];
+  const NOT_CHAR_TOKENS = [
+    ...(options.enableEscapeTags ? [BACKSLASH] : []),
+    openTag, SPACE, TAB, N, BACKSLASH,
+  ];
   const WHITESPACES = [SPACE, TAB];
   const SPECIAL_CHARS = [EQ, SPACE, TAB];
 
@@ -143,6 +147,7 @@ function createLexer(buffer, options = {}) {
 
   const next = () => {
     const currChar = bufferGrabber.getCurr();
+    const nextChar = bufferGrabber.getNext();
 
     if (currChar === N) {
       bufferGrabber.skip();
@@ -153,8 +158,12 @@ function createLexer(buffer, options = {}) {
     } else if (isWhiteSpace(currChar)) {
       const str = bufferGrabber.grabWhile(isWhiteSpace);
       emitToken(createToken(TYPE_SPACE, str, row, col));
+    } else if (options.enableEscapeTags && currChar === BACKSLASH
+               && (nextChar === openTag || nextChar === closeTag)) {
+      bufferGrabber.skip(); // skip the \ without emitting anything
+      bufferGrabber.skip(); // skip past the [ or ] as well
+      emitToken(createToken(TYPE_WORD, nextChar, row, col));
     } else if (currChar === openTag) {
-      const nextChar = bufferGrabber.getNext();
       bufferGrabber.skip(); // skip openTag
 
       // detect case where we have '[My word [tag][/tag]' or we have '[My last line word'

packages/bbob-parser/src/parse.js

@@ -10,6 +10,7 @@ import { createList } from './utils';
  * @param {Array<string>} opts.onlyAllowTags
  * @param {String} opts.openTag
  * @param {String} opts.closeTag
+ * @param {Boolean} opts.enableEscapeTags
  * @return {Array}
  */
 const parse = (input, opts = {}) => {
@@ -220,6 +221,7 @@ const parse = (input, opts = {}) => {
     onlyAllowTags: options.onlyAllowTags,
     openTag: options.openTag,
     closeTag: options.closeTag,
+    enableEscapeTags: options.enableEscapeTags,
   });
 
   // eslint-disable-next-line no-unused-vars

packages/bbob-parser/test/lexer.test.js

@@ -288,6 +288,23 @@ describe('lexer', () => {
     expectOutput(output, tokens);
   });
 
+  test('escaped tag', () => {
+    const tokenizeEscape = input => (createLexer(input, {
+      enableEscapeTags: true
+    }).tokenize());
+    const input = '\\[b\\]test\\[';
+    const tokens = tokenizeEscape(input);
+    const output = [
+      [TYPE.WORD, '[', '0', '0'],
+      [TYPE.WORD, 'b', '0', '0'],
+      [TYPE.WORD, ']', '0', '0'],
+      [TYPE.WORD, 'test', '0', '0'],
+      [TYPE.WORD, '[', '0', '0'],
+    ];
+
+    expectOutput(output, tokens);
+  });
+
   describe('html', () => {
     const tokenizeHTML = input => createLexer(input, { openTag: '<', closeTag: '>' }).tokenize();
 

packages/bbob-parser/test/parse.test.js

@@ -183,5 +183,21 @@ describe('Parser', () => {
         }
       ]);
     });
+
+    test('parse escaped tags tags', () => {
+      const ast = parse('\\[b\\]test\\[/b\\]', {
+        enableEscapeTags: true
+      });
+
+      expectOutput(ast, [
+        '[',
+        'b',
+        ']',
+        'test',
+        '[',
+        '/b',
+        ']',
+      ]);
+    });
   });
 });