feat(parser): rewrite lexer to make it faster (#50)

* feat(parser): first iteration of new lexer

* feat(parser): convert token string props to number props

* refactor(parser): optimize char grabber

* refactor(parser): working on new lexer

* refactor(parser): convert token string props to number props

* refactor(parser): rebuild lexer, add tag attrs parsing

* refactor(parser): rework word parsing and tag parsing

* refactor(parser): rework to pass tests

* refactor(parser): rework tag parsing

* refactor(parser): rework escape tags parsing

* refactor(parser): rework tests

* refactor(parser): all test pass

* refactor(parser): make lexer faster by move mode switching in loop

* refactor(parser): remove all state map objects

* refactor(parser): order of parsing states

* refactor(parser): state switching without return

* refactor(parser): rename buffers to chars

* refactor(lexer): reduce function calls

* feat(lexer): add new parser tests and code to pass it

* fix(utils): remove unused variable in char grabber

* feat(lexer): add test for new lexer bug

* chore(*): add lexer and lexer2 to benchmark

* chore(lexer): add some debug info for char grabber

* feat(parser): add new test for single attributes without values

* fix(lexer): paired tags tests

* refactor(lexer): comment breaking changes tests for future releases

* feat(core): improve tests

* refactor(parser): add more tests, reduce char grabber size

* refactor(parser): reduce utils size

* refactor(parser): remove unused code from tag parsing code

* refactor(parser): remove unused code from word to tag transforming code

* chore(benchmark): fix benchmark imports

benchmark/index.js

@@ -1,3 +1,4 @@
+/* eslint-disable global-require */
 const Benchmark = require('benchmark');
 const stub = require('./test/stub');
 
@@ -38,9 +39,22 @@ suite
       addInLineBreaks: false,
     });
   })
-  .add('@bbob/parser', () => require('../packages/bbob-parser/lib/index').parse(stub, {
-    onlyAllowTags: ['ch'],
-  }))
+  .add('@bbob/parser lexer old', () => {
+    const lexer1 = require('../packages/bbob-parser/lib/lexer_old');
+
+    return require('../packages/bbob-parser/lib/index').parse(stub, {
+      onlyAllowTags: ['ch'],
+      createTokenizer: lexer1.createLexer,
+    });
+  })
+  .add('@bbob/parser lexer', () => {
+    const lexer2 = require('../packages/bbob-parser/lib/lexer');
+
+    return require('../packages/bbob-parser/lib/index').parse(stub, {
+      onlyAllowTags: ['ch'],
+      createTokenizer: lexer2.createLexer,
+    });
+  })
 // add listeners
   .on('cycle', (event) => {
     console.log(String(event.target));