fix(parser): tokenizer error with quotemark strings

lerna.json

@@ -4,7 +4,8 @@
     "packages/*"
   ],
   "publishConfig": {
-    "access": "public"
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
   },
   "version": "independent"
 }

package.json

@@ -24,5 +24,9 @@
     "rollup-plugin-commonjs": "^9.1.3",
     "rollup-plugin-node-resolve": "^3.3.0",
     "xbbcode-parser": "^0.1.2"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
   }
 }

packages/bbob-parser/lib/Tokenizer.js

@@ -219,7 +219,11 @@ class Tokenizer {
         !isPrevBackslash) {
       this.flushAttrNames();
     } else if (!this.inTag()) {
-      this.wordToken[Token.VALUE_ID] += getChar(charCode);
+      if (!this.wordToken) {
+        this.wordToken = this.createWordToken(getChar(charCode));
+      } else {
+        this.wordToken[Token.VALUE_ID] += getChar(charCode);
+      }
     }
 
     this.nextCol();

packages/bbob-parser/lib/parse.js

@@ -167,9 +167,15 @@ const handleTagEnd = (token) => {
 
     if (lastNestedNode) {
       appendNode(lastNestedNode);
-    } else {
-      // eslint-disable-next-line no-console
-      console.warn(`Inconsistent tag '${getTokenValue(token)}' on line ${getTokenLine(token)} and column ${getTokenColumn(token)}`);
+    } else if (options.onError) {
+      const tag = getTokenValue(token);
+      const line = getTokenLine(token);
+      const column = getTokenColumn(token);
+      options.onError({
+        message: `Inconsistent tag '${tag}' on line ${line} and column ${column}`,
+        lineNumber: line,
+        columnNumber: column,
+      });
     }
   }
 };
@@ -228,8 +234,8 @@ const parseToken = (token) => {
  * @return {Array}
  */
 const parse = (input, opts = {}) => {
-  tokenizer = createTokenizer(input, parseToken);
   options = opts;
+  tokenizer = createTokenizer(input, parseToken);
 
   nodes = [];
   nestedNodes = [];

packages/bbob-parser/test/Tokenizer.test.js

@@ -34,6 +34,25 @@ describe('Tokenizer', () => {
     expectOutput(output, tokens);
   });
 
+  test('tokenize string with quotemarks', () => {
+    const input = '"Someone Like You" by Adele';
+    const tokens = tokenize(input);
+
+    const output = [
+      [TYPE.WORD, '"Someone', '0', '0'],
+      [TYPE.SPACE, ' ', '8', '0'],
+      [TYPE.WORD, 'Like', '8', '0'],
+      [TYPE.SPACE, ' ', '13', '0'],
+      [TYPE.WORD, 'You"', '13', '0'],
+      [TYPE.SPACE, ' ', '18', '0'],
+      [TYPE.WORD, 'by', '18', '0'],
+      [TYPE.SPACE, ' ', '21', '0'],
+      [TYPE.WORD, 'Adele', '21', '0'],
+    ];
+
+    expectOutput(output, tokens);
+  });
+
   test('tokenize tag as param', () => {
     const input = '[color="#ff0000"]Text[/color]';
     const tokens = tokenize(input);

packages/bbob-parser/test/parse.test.js

@@ -42,6 +42,30 @@ describe('Parser', () => {
     ]);
   });
 
+  test('parse inconsistent tags', () => {
+    const ast = parse('[h1 name=value]Foo [Bar] /h1]');
+
+    expect(ast).toBeInstanceOf(Array);
+    expect(ast).toEqual(
+      [
+        {
+          attrs: {},
+          tag: 'h1',
+          content: []
+        },
+        'Foo',
+        ' ',
+        {
+          tag: 'Bar',
+          attrs: {},
+          content: []
+        },
+        ' ',
+        '/h1',
+      ]
+    );
+  });
+
   test('parse tag with value param', () => {
     const ast = parse('[url=https://github.com/jilizart/bbob]BBob[/url]');