feat(#250): introduce caseFreeTags option (#251)

* chore: initial tests

* feat: parser test

* feat: add case free tags support

* fix: coverage upload

* fix: --disable=gcov

* fix: npm publish sha commit

* fix: change codecov to coveralls

* fix: change workflow pr build and publish

* chore: change coverage badge [skip ci]

.changeset/poor-pears-marry.md

@@ -0,0 +1,42 @@
+---
+"@bbob/parser": minor
+"@bbob/types": minor
+"@bbob/cli": minor
+"@bbob/core": minor
+"@bbob/html": minor
+"@bbob/plugin-helper": minor
+"@bbob/preset": minor
+"@bbob/preset-html5": minor
+"@bbob/preset-react": minor
+"@bbob/preset-vue": minor
+"@bbob/react": minor
+"@bbob/vue2": minor
+"@bbob/vue3": minor
+---
+
+New option flag `caseFreeTags` has been added
+
+This flag allows to parse case insensitive tags like `[h1]some[/H1]` -> `<h1>some</h1>`
+
+```js
+import html from '@bbob/html'
+import presetHTML5 from '@bbob/preset-html5'
+
+const processed = html(`[h1]some[/H1]`, presetHTML5(), { caseFreeTags: true })
+
+console.log(processed); // <h1>some</h1>
+```
+
+Also now you can pass `caseFreeTags` to `parse` function
+
+```js
+import { parse } from '@bbob/parser'
+
+const ast = parse('[h1]some[/H1]', {
+  caseFreeTags: true
+});
+```
+
+BREAKING CHANGE: `isTokenNested` function now accepts string `tokenValue` instead of `token`
+
+Changed codecov.io to coveralls.io for test coverage

.github/workflows/pr.yml

@@ -1,9 +1,16 @@
 name: Pull Request
 on:
+#  workflow_run:
+#    workflows:
+#      - Tests
+#      - Benchmark
+#    types:
+#      - completed
   pull_request:
     paths-ignore:
       - '.changeset/**'
       - '.husky/**'
+  workflow_dispatch:
 
 concurrency:
   group: ci-pull-request=${{github.ref}}-1
@@ -30,7 +37,7 @@ jobs:
       - name: Set SHA
         id: sha
         run: |
-          SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
+          SHORT_SHA=$(git rev-parse --short "${{ github.event.pull_request.head.sha }}")
           echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
 
       - name: Install pnpm

.github/workflows/test.yml

@@ -37,11 +37,8 @@ jobs:
       - name: Run the lint
         run: pnpm run lint
 
-      - name: Install coverage
-        run: pnpm install --global codecov
-
       - name: Run the coverage
         run: pnpm run cover
 
-      - name: Run the coverage
+      - name: Coveralls
-        run: codecov
+        uses: coverallsapp/github-action@v2

README.md

@@ -8,9 +8,9 @@ written in pure javascript, no dependencies
 
 [![Tests](https://github.com/JiLiZART/BBob/actions/workflows/test.yml/badge.svg)](https://github.com/JiLiZART/BBob/actions/workflows/test.yml)
 [![Benchmark](https://github.com/JiLiZART/BBob/actions/workflows/benchmark.yml/badge.svg)](https://github.com/JiLiZART/BBob/actions/workflows/benchmark.yml)
-<a href="https://codecov.io/gh/JiLiZART/bbob">
+<a href='https://coveralls.io/github/JiLiZART/BBob?branch=master'>
-  <img src="https://codecov.io/gh/JiLiZART/bbob/branch/master/graph/badge.svg" alt="codecov">
+    <img src='https://coveralls.io/repos/github/JiLiZART/BBob/badge.svg?branch=master' alt='Coverage Status' />
-</a> 
+</a>
 <a href="https://www.codefactor.io/repository/github/jilizart/bbob">
   <img src="https://www.codefactor.io/repository/github/jilizart/bbob/badge" alt="CodeFactor">
 </a>
@@ -230,6 +230,28 @@ const processed = bbobHTML(`[b]Text[/b]'\\[b\\]Text\\[/b\\]'`, presetHTML5(), {
 console.log(processed); // <span style="font-weight: bold;">Text</span>[b]Text[/b]
 ```
 
+#### caseFreeTags
+
+Allows to parse case insensitive tags like `[h1]some[/H1]` -> `<h1>some</h1>`
+
+```js
+import bbobHTML from '@bbob/html'
+import presetHTML5 from '@bbob/preset-html5'
+
+const processed = bbobHTML(`[h1]some[/H1]`, presetHTML5(), { caseFreeTags: true })
+
+console.log(processed); // <h1>some</h1>
+```
+
+```js
+import bbobHTML from '@bbob/html'
+import presetHTML5 from '@bbob/preset-html5'
+
+const processed = bbobHTML(`[b]Text[/b]'\\[b\\]Text\\[/b\\]'`, presetHTML5(), { enableEscapeTags: true })
+
+console.log(processed); // <span style="font-weight: bold;">Text</span>[b]Text[/b]
+```
+
 
 ### Presets <a name="basic"></a>
 

benchmark/package.json

@@ -10,8 +10,7 @@
     "cpupro": "node --require cpupro benchmark.js"
   },
   "author": {
-    "name": "Nikolay Kostyurin <jilizart@gmail.com>",
+    "name": "Nikolay Kostyurin <jilizart@gmail.com>"
-    "url": "https://artkost.ru/"
   },
   "dependencies": {
     "@bbob/parser": "*",

package.json

@@ -20,8 +20,7 @@
     "cleanup": "node scripts/cleanup"
   },
   "author": {
-    "name": "Nikolay Kostyurin <jilizart@gmail.com>",
+    "name": "Nikolay Kostyurin <jilizart@gmail.com>"
-    "url": "https://artkost.ru/"
   },
   "license": "MIT",
   "devDependencies": {

packages/bbob-parser/src/Token.ts

@@ -81,11 +81,11 @@ const getTagName = (token: Token) => {
   return isTagEnd(token) ? value.slice(1) : value;
 };
 
-const tokenToText = (token: Token) => {
+const tokenToText = (token: Token, openTag = OPEN_BRAKET, closeTag = CLOSE_BRAKET) => {
-  let text = OPEN_BRAKET;
+  let text = openTag;
 
   text += getTokenValue(token);
-  text += CLOSE_BRAKET;
+  text += closeTag;
 
   return text;
 };
@@ -167,8 +167,8 @@ class Token<TokenValue = string> implements TokenInterface {
     return getEndPosition(this);
   }
 
-  toString() {
+  toString({ openTag = OPEN_BRAKET, closeTag = CLOSE_BRAKET } = {}) {
-    return tokenToText(this);
+    return tokenToText(this, openTag, closeTag);
   }
 }
 

packages/bbob-parser/src/lexer.ts

@@ -51,13 +51,14 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
   let stateMode = STATE_WORD;
   let tagMode = TAG_STATE_NAME;
   let contextFreeTag = '';
-  const tokens = new Array<Token<string>>(Math.floor(buffer.length));
+  const tokens = new Array<Token>(Math.floor(buffer.length));
   const openTag = options.openTag || OPEN_BRAKET;
   const closeTag = options.closeTag || CLOSE_BRAKET;
   const escapeTags = !!options.enableEscapeTags;
   const contextFreeTags = (options.contextFreeTags || [])
     .filter(Boolean)
     .map((tag) => tag.toLowerCase());
+  const caseFreeTags = options.caseFreeTags || false;
   const nestedMap = new Map<string, boolean>();
   const onToken = options.onToken || (() => {
   });
@@ -88,8 +89,6 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
 
   /**
    * Emits newly created token to subscriber
-   * @param {Number} type
-   * @param {String} value
    */
   function emitToken(type: number, value: string, startPos?: number, endPos?: number) {
     const token = createTokenOfType(type, value, row, prevCol, startPos, endPos);
@@ -352,13 +351,13 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
     return tokens;
   }
 
-  function isTokenNested(token: Token) {
+  function isTokenNested(tokenValue: string) {
-    const value = openTag + SLASH + token.getValue();
+    const value = openTag + SLASH + tokenValue;
 
     if (nestedMap.has(value)) {
       return !!nestedMap.get(value);
     } else {
-      const status = (buffer.indexOf(value) > -1);
+      const status = caseFreeTags ? (buffer.toLowerCase().indexOf(value.toLowerCase()) > -1) : (buffer.indexOf(value) > -1);
 
       nestedMap.set(value, status);
 

packages/bbob-parser/src/parse.ts

@@ -52,8 +52,9 @@ function parse(input: string, opts: ParseOptions = {}) {
   const onlyAllowTags = (options.onlyAllowTags || [])
     .filter(Boolean)
     .map((tag) => tag.toLowerCase());
+  const caseFreeTags = options.caseFreeTags || false;
 
-  let tokenizer: LexerTokenizer | null = null;
+  let tokenizer: ReturnType<typeof createLexer> | null = null;
 
   /**
    * Result AST of nodes
@@ -85,10 +86,11 @@ function parse(input: string, opts: ParseOptions = {}) {
   const nestedTagsMap = new Set<string>();
 
   function isTokenNested(token: Token) {
-    const value = token.getValue();
+    const tokenValue = token.getValue();
+    const value = caseFreeTags ? tokenValue.toLowerCase() : tokenValue;
     const { isTokenNested } = tokenizer || {};
 
-    if (!nestedTagsMap.has(value) && isTokenNested && isTokenNested(token)) {
+    if (!nestedTagsMap.has(value) && isTokenNested && isTokenNested(value)) {
       nestedTagsMap.add(value);
 
       return true;
@@ -101,7 +103,7 @@ function parse(input: string, opts: ParseOptions = {}) {
    * @private
    */
   function isTagNested(tagName: string) {
-    return Boolean(nestedTagsMap.has(tagName));
+    return Boolean(nestedTagsMap.has(caseFreeTags ? tagName.toLowerCase() : tagName));
   }
 
   /**
@@ -203,17 +205,23 @@ function parse(input: string, opts: ParseOptions = {}) {
    * @param {Token} token
    */
   function handleTagEnd(token: Token) {
-    const lastTagNode = nestedNodes.last();
+    const tagName = token.getValue().slice(1);
-    if (isTagNode(lastTagNode)) {
-      lastTagNode.setEnd({ from: token.getStart(), to: token.getEnd() });
-    }
-    flushTagNodes();
-
     const lastNestedNode = nestedNodes.flush();
 
+    flushTagNodes();
+
     if (lastNestedNode) {
       const nodes = getNodes();
+
+      if (isTagNode(lastNestedNode)) {
+        lastNestedNode.setEnd({ from: token.getStart(), to: token.getEnd() });
+      }
+
       appendNodes(nodes, lastNestedNode);
+    } else if (!isTagNested(tagName)) { // when we have only close tag [/some] without any open tag
+      const nodes = getNodes();
+
+      appendNodes(nodes, token.toString({ openTag, closeTag }));
     } else if (typeof options.onError === "function") {
       const tag = token.getValue();
       const line = token.getLine();
@@ -281,13 +289,13 @@ function parse(input: string, opts: ParseOptions = {}) {
         }
       } else if (token.isTag()) {
         // if tag is not allowed, just pass it as is
-        appendNodes(nodes, token.toString());
+        appendNodes(nodes, token.toString({ openTag, closeTag }));
       }
     } else if (token.isText()) {
       appendNodes(nodes, tokenValue);
     } else if (token.isTag()) {
       // if tag is not allowed, just pass it as is
-      appendNodes(nodes, token.toString());
+      appendNodes(nodes, token.toString({ openTag, closeTag }));
     }
   }
 
@@ -311,6 +319,7 @@ function parse(input: string, opts: ParseOptions = {}) {
     closeTag,
     onlyAllowTags: options.onlyAllowTags,
     contextFreeTags: options.contextFreeTags,
+    caseFreeTags: options.caseFreeTags,
     enableEscapeTags: options.enableEscapeTags,
   });
 

packages/bbob-parser/test/parse.test.ts

@@ -247,6 +247,58 @@ describe('Parser', () => {
     });
   });
 
+  describe('caseFreeTags', () => {
+    test('default case tags', () => {
+      const ast = parse('[h1 name=value]Foo[/H1]', {
+        caseFreeTags: false
+      });
+      const output = [
+        {
+          tag: 'h1',
+          attrs: {
+            name: 'value'
+          },
+          content: [],
+          start: {
+            from: 0,
+            to: 15,
+          }
+        },
+        "Foo",
+        "[/H1]"
+      ];
+
+      expectOutput(ast, output);
+    });
+
+    test('case free tags', () => {
+      const ast = parse('[h1 name=value]Foo[/H1]', {
+        caseFreeTags: true
+      });
+      const output = [
+        {
+          tag: 'h1',
+          attrs: {
+            name: 'value'
+          },
+          content: [
+            "Foo"
+          ],
+          start: {
+            from: 0,
+            to: 15,
+          },
+          end: {
+            from: 18,
+            to: 23,
+          },
+        }
+      ];
+
+      expectOutput(ast, output);
+    });
+  })
+
   test('parse inconsistent tags', () => {
     const ast = parse('[h1 name=value]Foo [Bar] /h1]');
     const output = [
@@ -279,6 +331,15 @@ describe('Parser', () => {
     expectOutput(ast, output);
   });
 
+  test('parse closed tag', () => {
+    const ast = parse('[/h1]');
+    const output = [
+      '[/h1]',
+    ];
+
+    expectOutput(ast, output);
+  });
+
   test('parse tag with value param', () => {
     const ast = parse('[url=https://github.com/jilizart/bbob]BBob[/url]');
     const output = [
@@ -650,49 +711,49 @@ sdfasdfasdf
 [url=xxx]xxx[/url]`;
 
     expectOutput(
-      parse(str),
+        parse(str),
-      [
+        [
-        {
+          {
-          tag: 'quote', attrs: {}, content: ['some'],
+            tag: 'quote', attrs: {}, content: ['some'],
-          start: {
+            start: {
-            from: 0,
+              from: 0,
-            to: 7,
+              to: 7,
+            },
+            end: {
+              from: 11,
+              to: 19,
+            },
           },
-          end: {
+          {
-            from: 11,
+            tag: 'color', attrs: { red: 'red' }, content: ['test'],
-            to: 19,
+            start: {
+              from: 19,
+              to: 30,
+            },
+            end: {
+              from: 34,
+              to: 42,
+            },
           },
-        },
+          '\n',
-        {
+          '[quote]',
-          tag: 'color', attrs: { red: 'red' }, content: ['test'],
+          'xxxsdfasdf',
-          start: {
+          '\n',
-            from: 19,
+          'sdfasdfasdf',
-            to: 30,
+          '\n',
-          },
+          '\n',
-          end: {
+          {
-            from: 34,
+            tag: 'url', attrs: { xxx: 'xxx' }, content: ['xxx'],
-            to: 42,
+            start: {
-          },
+              from: 74,
-        },
+              to: 83,
-        '\n',
+            },
-        '[quote]',
+            end: {
-        'xxxsdfasdf',
+              from: 86,
-        '\n',
+              to: 92,
-        'sdfasdfasdf',
+            },
-        '\n',
+          }
-        '\n',
+        ]
-        {
-          tag: 'url', attrs: { xxx: 'xxx' }, content: ['xxx'],
-          start: {
-            from: 74,
-            to: 83,
-          },
-          end: {
-            from: 86,
-            to: 92,
-          },
-        }
-      ]
     );
   });
 
@@ -700,45 +761,45 @@ sdfasdfasdf
     const str = `[quote]xxxsdfasdf[quote]some[/quote][color=red]test[/color]sdfasdfasdf[url=xxx]xxx[/url]`;
 
     expectOutput(
-      parse(str),
+        parse(str),
-      [
+        [
-        '[quote]',
+          '[quote]',
-        'xxxsdfasdf',
+          'xxxsdfasdf',
-        {
+          {
-          tag: 'quote', attrs: {}, content: ['some'],
+            tag: 'quote', attrs: {}, content: ['some'],
-          start: {
+            start: {
-            from: 17,
+              from: 17,
-            to: 24,
+              to: 24,
+            },
+            end: {
+              from: 28,
+              to: 36,
+            },
           },
-          end: {
+          {
-            from: 28,
+            tag: 'color', attrs: { red: 'red' }, content: ['test'],
-            to: 36,
+            start: {
+              from: 36,
+              to: 47,
+            },
+            end: {
+              from: 51,
+              to: 59,
+            },
           },
-        },
+          'sdfasdfasdf',
-        {
+          {
-          tag: 'color', attrs: { red: 'red' }, content: ['test'],
+            tag: 'url', attrs: { xxx: 'xxx' }, content: ['xxx'],
-          start: {
+            start: {
-            from: 36,
+              from: 70,
-            to: 47,
+              to: 79,
-          },
+            },
-          end: {
+            end: {
-            from: 51,
+              from: 82,
-            to: 59,
+              to: 88,
-          },
+            },
-        },
+          }
-        'sdfasdfasdf',
+        ]
-        {
-          tag: 'url', attrs: { xxx: 'xxx' }, content: ['xxx'],
-          start: {
-            from: 70,
-            to: 79,
-          },
-          end: {
-            from: 82,
-            to: 88,
-          },
-        }
-      ]
     );
   });
 
@@ -746,45 +807,45 @@ sdfasdfasdf
     const str = `[quote]some[/quote][color=red]test[/color]sdfasdfasdf[url=xxx]xxx[/url][quote]xxxsdfasdf`;
 
     expectOutput(
-      parse(str),
+        parse(str),
-      [
+        [
-        {
+          {
-          tag: 'quote', attrs: {}, content: ['some'],
+            tag: 'quote', attrs: {}, content: ['some'],
-          start: {
+            start: {
-            from: 0,
+              from: 0,
-            to: 7,
+              to: 7,
+            },
+            end: {
+              from: 11,
+              to: 19,
+            },
           },
-          end: {
+          {
-            from: 11,
+            tag: 'color', attrs: { red: 'red' }, content: ['test'],
-            to: 19,
+            start: {
+              from: 19,
+              to: 30,
+            },
+            end: {
+              from: 34,
+              to: 42,
+            },
           },
-        },
+          'sdfasdfasdf',
-        {
+          {
-          tag: 'color', attrs: { red: 'red' }, content: ['test'],
+            tag: 'url', attrs: { xxx: 'xxx' }, content: ['xxx'],
-          start: {
+            start: {
-            from: 19,
+              from: 53,
-            to: 30,
+              to: 62,
+            },
+            end: {
+              from: 65,
+              to: 71,
+            },
           },
-          end: {
+          '[quote]',
-            from: 34,
+          'xxxsdfasdf',
-            to: 42,
+        ]
-          },
-        },
-        'sdfasdfasdf',
-        {
-          tag: 'url', attrs: { xxx: 'xxx' }, content: ['xxx'],
-          start: {
-            from: 53,
-            to: 62,
-          },
-          end: {
-            from: 65,
-            to: 71,
-          },
-        },
-        '[quote]',
-        'xxxsdfasdf',
-      ]
     );
   });
 

packages/bbob-types/src/parser.ts

@@ -23,24 +23,23 @@ export interface Token<TokenValue = string> {
 
 export interface LexerTokenizer {
   tokenize: () => Token<string>[];
-  isTokenNested?: (token: Token<string>) => boolean;
+  isTokenNested?: (tokenValue: string) => boolean;
 }
 
-export interface LexerOptions {
+export interface CommonOptions {
   openTag?: string;
   closeTag?: string;
   onlyAllowTags?: string[];
   enableEscapeTags?: boolean;
+  caseFreeTags?: boolean;
   contextFreeTags?: string[];
+}
+
+export interface LexerOptions extends CommonOptions {
   onToken?: (token?: Token<string>) => void;
 }
 
-export interface ParseOptions {
+export interface ParseOptions extends CommonOptions {
   createTokenizer?: (input: string, options?: LexerOptions) => LexerTokenizer;
-  openTag?: string;
-  closeTag?: string;
-  onlyAllowTags?: string[];
-  contextFreeTags?: string[];
-  enableEscapeTags?: boolean;
   onError?: (error: ParseError) => void;
 }

scripts/package.json

@@ -5,7 +5,6 @@
     "pkg-task": "pkg-task"
   },
   "author": {
-    "name": "Nikolay Kostyurin <jilizart@gmail.com>",
+    "name": "Nikolay Kostyurin <jilizart@gmail.com>"
-    "url": "https://artkost.ru/"
   }
 }