fix parse tags in brackets

packages/bbob-parser/lib/Tokenizer.js

@@ -177,14 +177,30 @@ class Tokenizer {
     this.colPos = 0;
   }
 
-  charOPENBRAKET() {
-    this.flushWord();
-    this.tagToken = this.createTagToken('');
+  charOPENBRAKET(charCode) {
+    const nextCharCode = this.seekChar(1);
+    const isNextSpace = nextCharCode === SPACE || nextCharCode === TAB;
+
+    if (isNextSpace) {
+      this.createWord();
+      this.wordToken[Token.VALUE_ID] += getChar(charCode);
+    } else {
+      this.flushWord();
+
+      this.tagToken = this.createTagToken('');
+    }
 
     this.nextCol();
   }
 
-  charCLOSEBRAKET() {
+  charCLOSEBRAKET(charCode) {
+    const prevCharCode = this.seekChar(-1);
+    const isPrevSpace = prevCharCode === SPACE || prevCharCode === TAB;
+
+    if (isPrevSpace) {
+      this.wordToken[Token.VALUE_ID] += getChar(charCode);
+    }
+
     this.nextCol();
     this.flushTag();
     this.flushAttrNames();
@@ -328,6 +344,8 @@ class Tokenizer {
   }
 }
 
+new Tokenizer('[ [g]G[/g] ]').tokenize()
+
 module.exports = Tokenizer;
 module.exports.createTokenOfType = createTokenOfType;
 module.exports.TYPE = {

packages/bbob-parser/test/Tokenizer.test.js

@@ -53,6 +53,23 @@ describe('Tokenizer', () => {
     expectOutput(output, tokens);
   });
 
+  test('tokenize tags in brakets', () => {
+    const input = '[ [h1]G[/h1] ]';
+    const tokens = tokenize(input);
+
+    const output = [
+      [TYPE.WORD, '[', '0', '0'],
+      [TYPE.SPACE, ' ', '1', '0'],
+      [TYPE.TAG, 'h1', '2', '0'],
+      [TYPE.WORD, 'G', '1', '0'],
+      [TYPE.TAG, '/h1', '7', '0'],
+      [TYPE.SPACE, ' ', '12', '0'],
+      [TYPE.WORD, ']', '7', '0'],
+    ];
+
+    expectOutput(output, tokens);
+  });
+
   test('tokenize tag as param', () => {
     const input = '[color="#ff0000"]Text[/color]';
     const tokens = tokenize(input);