move test files and lib files to separate folders

packages/bbob-parser/lib/Parser.js

@@ -0,0 +1,201 @@
+const {
+  convertTokenToText,
+  getTagName,
+  getTokenColumn,
+  getTokenLine,
+  getTokenValue,
+  isAttrNameToken,
+  isAttrValueToken,
+  isTagStart,
+  isTagToken,
+  isTextToken,
+  isTagEnd,
+} = require('./token');
+
+const {
+  SLASH,
+  getChar,
+} = require('./char');
+
+const createTagNode = (tag, attrs = {}, content = []) => ({ tag, attrs, content });
+
+/**
+ *
+ {
+     tag: 'div',
+     attrs: {
+         class: 'foo'
+     },
+     content: ['hello world!']
+ }
+ */
+module.exports = class Parser {
+  constructor(tokens, options = {}) {
+    this.tokens = tokens;
+    this.options = options;
+
+    this.closableTags = this.findNestedTags();
+    this.nodes = [];
+    this.nestedNodes = [];
+    this.curTags = [];
+    this.curTagsAttrName = [];
+  }
+
+  isNestedTag(token) {
+    return this.closableTags.indexOf(getTokenValue(token)) >= 0;
+  }
+
+  getCurTag() {
+    if (this.curTags.length) {
+      return this.curTags[this.curTags.length - 1];
+    }
+
+    return null;
+  }
+
+  createCurTag(token) {
+    this.curTags.push(createTagNode(getTokenValue(token)));
+  }
+
+  createCurTagAttrName(token) {
+    this.curTagsAttrName.push(getTokenValue(token));
+  }
+
+  getCurTagAttrName() {
+    if (this.curTagsAttrName.length) {
+      return this.curTagsAttrName[this.curTagsAttrName.length - 1];
+    }
+
+    return this.getCurTag().tag;
+  }
+
+  clearCurTagAttrName() {
+    if (this.curTagsAttrName.length) {
+      this.curTagsAttrName.pop();
+    }
+  }
+
+  clearCurTag() {
+    if (this.curTags.length) {
+      this.curTags.pop();
+
+      this.clearCurTagAttrName();
+    }
+  }
+
+  getNodes() {
+    if (this.nestedNodes.length) {
+      const nestedNode = this.nestedNodes[this.nestedNodes.length - 1];
+      return nestedNode.content;
+    }
+
+    return this.nodes;
+  }
+
+  appendNode(tag) {
+    this.getNodes().push(tag);
+  }
+
+  handleTagStart(token) {
+    if (isTagStart(token)) {
+      this.createCurTag(token);
+
+      if (this.isNestedTag(token)) {
+        this.nestedNodes.push(this.getCurTag());
+      } else {
+        this.appendNode(this.getCurTag());
+        this.clearCurTag();
+      }
+    }
+  }
+
+  handleTagEnd(token) {
+    if (isTagEnd(token)) {
+      this.clearCurTag();
+
+      const lastNestedNode = this.nestedNodes.pop();
+
+      if (lastNestedNode) {
+        this.appendNode(lastNestedNode);
+      } else {
+        // eslint-disable-next-line no-console
+        console.warn(`Inconsistent tag '${getTokenValue(token)}' on line ${getTokenLine(token)} and column ${getTokenColumn(token)}`);
+      }
+    }
+  }
+
+  handleTagToken(token) {
+    if (isTagToken(token)) {
+      if (this.isAllowedTag(getTagName(token))) {
+        // [tag]
+        this.handleTagStart(token);
+
+        // [/tag]
+        this.handleTagEnd(token);
+      } else {
+        this.appendNode(convertTokenToText(token));
+      }
+    }
+  }
+
+  handleCurTag(token) {
+    if (this.getCurTag()) {
+      if (isAttrNameToken(token)) {
+        this.createCurTagAttrName(token);
+        this.getCurTag().attrs[this.getCurTagAttrName()] = null;
+      } else if (isAttrValueToken(token)) {
+        this.getCurTag().attrs[this.getCurTagAttrName()] = getTokenValue(token);
+        this.clearCurTagAttrName();
+      } else if (isTextToken(token)) {
+        this.getCurTag().content.push(getTokenValue(token));
+      }
+    } else if (isTextToken(token)) {
+      this.appendNode(getTokenValue(token));
+    }
+  }
+
+  parse() {
+    let token;
+    // eslint-disable-next-line no-cond-assign
+    while (token = this.tokens.shift()) {
+      if (!token) {
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+
+      this.handleTagToken(token);
+
+      this.handleCurTag(token);
+    }
+
+    return this.nodes;
+  }
+
+  findNestedTags() {
+    const tags = this.tokens.filter(isTagToken).reduce((acc, token) => {
+      acc[getTokenValue(token)] = true;
+
+      return acc;
+    }, {});
+
+    const closeChar = getChar(SLASH);
+
+    return Object.keys(tags).reduce((arr, key) => {
+      if (tags[key] && tags[closeChar + key]) {
+        arr.push(key);
+      }
+
+      return arr;
+    }, []);
+  }
+
+  isAllowedTag(value) {
+    if (this.options.onlyAllowTags && this.options.onlyAllowTags.length) {
+      return this.options.onlyAllowTags.indexOf(value) >= 0;
+    }
+
+    return true;
+  }
+};
+
+module.exports.createTagNode = createTagNode;

packages/bbob-parser/lib/Tokenizer.js

@@ -0,0 +1,283 @@
+const {
+  getChar,
+  OPEN_BRAKET,
+  CLOSE_BRAKET, EQ, TAB, SPACE, N, QUOTEMARK,
+  PLACEHOLDER_SPACE, PLACEHOLDER_SPACE_TAB,
+} = require('./char');
+const TOKEN = require('./token');
+
+class Tokenizer {
+  constructor(input) {
+    this.buffer = input;
+    this.colPos = 0;
+    this.rowPos = 0;
+    this.index = 0;
+
+    this.tokenIndex = -1;
+    this.tokens = new Array(Math.floor(this.buffer.length));
+    this.dummyArray = ['', '', '', ''];
+
+    this.wordToken = this.dummyArray;
+    this.tagToken = this.dummyArray;
+    this.attrNameToken = this.dummyArray;
+    this.attrValueToken = this.dummyArray;
+    this.attrTokens = [];
+  }
+
+  appendToken(token) {
+    this.tokenIndex += 1;
+    this.tokens[this.tokenIndex] = token;
+  }
+
+  nextCol() {
+    this.colPos += 1;
+  }
+
+  nextLine() {
+    this.rowPos += 1;
+  }
+
+  flushWord() {
+    if (this.wordToken[TOKEN.TYPE_ID] && this.wordToken[TOKEN.VALUE_ID]) {
+      this.appendToken(this.wordToken);
+      this.wordToken = this.createWordToken('');
+    }
+  }
+
+  createWord(value, line, row) {
+    if (this.wordToken[TOKEN.TYPE_ID] === '') {
+      this.wordToken = this.createWordToken(value, line, row);
+    }
+  }
+
+  flushTag() {
+    if (this.tagToken[TOKEN.TYPE_ID]) {
+      // [] and [=] tag case
+      if (this.tagToken[TOKEN.VALUE_ID] === '') {
+        const value = this.attrValueToken[TOKEN.TYPE_ID] ? getChar(EQ) : '';
+        const word = getChar(OPEN_BRAKET) + value + getChar(CLOSE_BRAKET);
+
+        this.createWord('', 0, 0);
+        this.wordToken[TOKEN.VALUE_ID] += word;
+
+        this.tagToken = this.dummyArray;
+
+        if (this.attrValueToken[TOKEN.TYPE_ID]) {
+          this.attrValueToken = this.dummyArray;
+        }
+
+        return;
+      }
+
+      if (this.attrNameToken[TOKEN.TYPE_ID] && !this.attrValueToken[TOKEN.TYPE_ID]) {
+        this.tagToken[TOKEN.VALUE_ID] += PLACEHOLDER_SPACE + this.attrNameToken[TOKEN.VALUE_ID];
+        this.attrNameToken = this.dummyArray;
+      }
+
+      this.appendToken(this.tagToken);
+      this.tagToken = this.dummyArray;
+    }
+  }
+
+  flushUnclosedTag() {
+    if (this.tagToken[TOKEN.TYPE_ID]) {
+      const value = this.tagToken[TOKEN.VALUE_ID] + (this.attrValueToken[TOKEN.VALUE_ID] ? getChar(EQ) : '');
+
+      this.tagToken[TOKEN.TYPE_ID] = TOKEN.TYPE_WORD;
+      this.tagToken[TOKEN.VALUE_ID] = getChar(OPEN_BRAKET) + value;
+
+      this.appendToken(this.tagToken);
+
+      this.tagToken = this.dummyArray;
+
+      if (this.attrValueToken[TOKEN.TYPE_ID]) {
+        this.attrValueToken = this.dummyArray;
+      }
+    }
+  }
+
+  flushAttrNames() {
+    if (this.attrNameToken[TOKEN.TYPE_ID]) {
+      this.attrTokens.push(this.attrNameToken);
+      this.attrNameToken = this.dummyArray;
+    }
+
+    if (this.attrValueToken[TOKEN.TYPE_ID]) {
+      this.attrTokens.push(this.attrValueToken);
+      this.attrValueToken = this.dummyArray;
+    }
+  }
+
+  flushAttrs() {
+    if (this.attrTokens.length) {
+      this.attrTokens.forEach(this.appendToken.bind(this));
+      this.attrTokens = [];
+    }
+  }
+
+  charSPACE(charCode) {
+    this.flushWord();
+
+    if (this.tagToken[TOKEN.TYPE_ID]) {
+      this.attrNameToken = this.createAttrNameToken('');
+    } else {
+      const spaceCode = charCode === TAB ? PLACEHOLDER_SPACE_TAB : PLACEHOLDER_SPACE;
+
+      this.appendToken(this.createSpaceToken(spaceCode));
+    }
+    this.nextCol();
+  }
+
+  charN(charCode) {
+    this.flushWord();
+    this.appendToken(this.createNewLineToken(getChar(charCode)));
+
+    this.nextLine();
+    this.colPos = 0;
+  }
+
+  charOPENBRAKET() {
+    this.flushWord();
+    this.tagToken = this.createTagToken('');
+
+    this.nextCol();
+  }
+
+  charCLOSEBRAKET() {
+    this.flushTag();
+    this.flushAttrNames();
+    this.flushAttrs();
+
+    this.nextCol();
+  }
+
+  charEQ(charCode) {
+    if (this.tagToken[TOKEN.TYPE_ID]) {
+      this.attrValueToken = this.createAttrValueToken('');
+    } else {
+      this.wordToken[TOKEN.VALUE_ID] += getChar(charCode);
+    }
+
+    this.nextCol();
+  }
+
+  charQUOTEMARK(charCode) {
+    if (this.attrValueToken[TOKEN.TYPE_ID] && this.attrValueToken[TOKEN.VALUE_ID] > 0) {
+      this.flushAttrNames();
+    } else if (this.tagToken[TOKEN.TYPE_ID] === '') {
+      this.wordToken[TOKEN.VALUE_ID] += getChar(charCode);
+    }
+
+    this.nextCol();
+  }
+
+  charWORD(charCode) {
+    if (this.tagToken[TOKEN.TYPE_ID] && this.attrValueToken[TOKEN.TYPE_ID]) {
+      this.attrValueToken[TOKEN.VALUE_ID] += getChar(charCode);
+    } else if (this.tagToken[TOKEN.TYPE_ID] && this.attrNameToken[TOKEN.TYPE_ID]) {
+      this.attrNameToken[TOKEN.VALUE_ID] += getChar(charCode);
+    } else if (this.tagToken[TOKEN.TYPE_ID]) {
+      this.tagToken[TOKEN.VALUE_ID] += getChar(charCode);
+    } else {
+      this.createWord();
+
+      this.wordToken[TOKEN.VALUE_ID] += getChar(charCode);
+    }
+
+    this.nextCol();
+  }
+
+  tokenize() {
+    while (this.index < this.buffer.length) {
+      const charCode = this.buffer.charCodeAt(this.index);
+
+      switch (charCode) {
+        case TAB:
+        case SPACE:
+          this.charSPACE(charCode);
+          break;
+
+        case N:
+          this.charN(charCode);
+          break;
+
+        case OPEN_BRAKET:
+          this.charOPENBRAKET();
+          break;
+
+        case CLOSE_BRAKET:
+          this.charCLOSEBRAKET();
+          break;
+
+        case EQ:
+          this.charEQ(charCode);
+          break;
+
+        case QUOTEMARK:
+          this.charQUOTEMARK(charCode);
+          break;
+
+        default:
+          this.charWORD(charCode);
+          break;
+      }
+
+      this.index += 1;
+    }
+
+    this.flushWord();
+    this.flushUnclosedTag();
+
+    this.tokens.length = this.tokenIndex + 1;
+
+    return this.tokens;
+  }
+
+  createWordToken(value = '', line = this.colPos, row = this.rowPos) {
+    return this.createTokenOfType(TOKEN.TYPE_WORD, value, line, row);
+  }
+
+  createTagToken(value, line = this.colPos, row = this.rowPos) {
+    return this.createTokenOfType(TOKEN.TYPE_TAG, value, line, row);
+  }
+
+  createAttrNameToken(value, line = this.colPos, row = this.rowPos) {
+    return this.createTokenOfType(TOKEN.TYPE_ATTR_NAME, value, line, row);
+  }
+
+  createAttrValueToken(value, line = this.colPos, row = this.rowPos) {
+    return this.createTokenOfType(TOKEN.TYPE_ATTR_VALUE, value, line, row);
+  }
+
+  createSpaceToken(value, line = this.colPos, row = this.rowPos) {
+    return this.createTokenOfType(TOKEN.TYPE_SPACE, value, line, row);
+  }
+
+  createNewLineToken(value, line = this.colPos, row = this.rowPos) {
+    return this.createTokenOfType(TOKEN.TYPE_NEW_LINE, value, line, row);
+  }
+
+  createTokenOfType(type, value, line = this.colPos, row = this.rowPos) {
+    return [String(type), String(value), String(line), String(row)];
+  }
+}
+
+// warm up tokenizer to elimitate code branches that never execute
+new Tokenizer('[b param="hello"]Sample text[/b]\n\t[Chorus 2] x html([a. title][, alt][, classes]) x [=] [/y]').tokenize();
+
+module.exports = Tokenizer;
+module.exports.TYPE = {
+  WORD: TOKEN.TYPE_WORD,
+  TAG: TOKEN.TYPE_TAG,
+  ATTR_NAME: TOKEN.TYPE_ATTR_NAME,
+  ATTR_VALUE: TOKEN.TYPE_ATTR_VALUE,
+  SPACE: TOKEN.TYPE_SPACE,
+  NEW_LINE: TOKEN.TYPE_NEW_LINE,
+};
+module.exports.TOKEN = {
+  TYPE_ID: TOKEN.TYPE_ID,
+  VALUE_ID: TOKEN.VALUE_ID,
+  LINE_ID: TOKEN.LINE_ID,
+  COLUMN_ID: TOKEN.COLUMN_ID,
+};
+

packages/bbob-parser/lib/char.js

@@ -0,0 +1,34 @@
+const N = '\n'.charCodeAt(0);
+const TAB = '\t'.charCodeAt(0);
+const F = '\f'.charCodeAt(0);
+const R = '\r'.charCodeAt(0);
+
+const EQ = '='.charCodeAt(0);
+const QUOTEMARK = '"'.charCodeAt(0);
+const SPACE = ' '.charCodeAt(0);
+
+const OPEN_BRAKET = '['.charCodeAt(0);
+const CLOSE_BRAKET = ']'.charCodeAt(0);
+
+const SLASH = '/'.charCodeAt(0);
+
+const PLACEHOLDER_SPACE_TAB = '    ';
+const PLACEHOLDER_SPACE = ' ';
+
+const getChar = String.fromCharCode;
+
+module.exports = {
+  getChar,
+  N,
+  F,
+  R,
+  TAB,
+  EQ,
+  QUOTEMARK,
+  SPACE,
+  OPEN_BRAKET,
+  CLOSE_BRAKET,
+  SLASH,
+  PLACEHOLDER_SPACE_TAB,
+  PLACEHOLDER_SPACE,
+};

packages/bbob-parser/lib/index.js

@@ -0,0 +1 @@
+module.exports = require('./parse');

packages/bbob-parser/lib/parse.js

@@ -0,0 +1,11 @@
+const Tokenizer = require('./Tokenizer');
+const Parser = require('./Parser');
+
+module.exports = function parse(input, options) {
+  const tokenizer = new Tokenizer(input);
+  const tokens = tokenizer.tokenize();
+  const parser = new Parser(tokens, options);
+  const ast = parser.parse();
+
+  return ast;
+};

packages/bbob-parser/lib/token.js

@@ -0,0 +1,79 @@
+const {
+  getChar,
+  OPEN_BRAKET,
+  CLOSE_BRAKET,
+  SLASH,
+} = require('./char');
+
+const TOKEN_TYPE_ID = 0;
+const TOKEN_VALUE_ID = 1;
+const TOKEN_COLUMN_ID = 2;
+const TOKEN_LINE_ID = 3;
+
+const TOKEN_TYPE_WORD = 'word';
+const TOKEN_TYPE_TAG = 'tag';
+const TOKEN_TYPE_ATTR_NAME = 'attr-name';
+const TOKEN_TYPE_ATTR_VALUE = 'attr-value';
+const TOKEN_TYPE_SPACE = 'space';
+const TOKEN_TYPE_NEW_LINE = 'new-line';
+
+const getTokenValue = token => token[TOKEN_VALUE_ID];
+const getTokenLine = token => token[TOKEN_LINE_ID];
+const getTokenColumn = token => token[TOKEN_COLUMN_ID];
+
+const isTextToken = (token) => {
+  const type = token[TOKEN_TYPE_ID];
+
+  return type === TOKEN_TYPE_SPACE || type === TOKEN_TYPE_NEW_LINE || type === TOKEN_TYPE_WORD;
+};
+
+const isTagToken = token => token[TOKEN_TYPE_ID] === TOKEN_TYPE_TAG;
+const isTagEnd = token => getTokenValue(token).charCodeAt(0) === SLASH;
+const isTagStart = token => !isTagEnd(token);
+const isAttrNameToken = token => token[TOKEN_TYPE_ID] === TOKEN_TYPE_ATTR_NAME;
+const isAttrValueToken = token => token[TOKEN_TYPE_ID] === TOKEN_TYPE_ATTR_VALUE;
+
+const getTagName = (token) => {
+  const value = getTokenValue(token);
+
+  return isTagEnd(token) ? value.slice(1) : value;
+};
+
+
+const convertTagToText = (token) => {
+  let text = getChar(OPEN_BRAKET);
+
+  if (isTagEnd(token)) {
+    text += getChar(SLASH);
+  }
+
+  text += getTokenValue(token);
+  text += getChar(CLOSE_BRAKET);
+
+  return text;
+};
+
+
+module.exports = {
+  TYPE_ID: TOKEN_TYPE_ID,
+  VALUE_ID: TOKEN_VALUE_ID,
+  LINE_ID: TOKEN_LINE_ID,
+  COLUMN_ID: TOKEN_COLUMN_ID,
+  TYPE_WORD: TOKEN_TYPE_WORD,
+  TYPE_TAG: TOKEN_TYPE_TAG,
+  TYPE_ATTR_NAME: TOKEN_TYPE_ATTR_NAME,
+  TYPE_ATTR_VALUE: TOKEN_TYPE_ATTR_VALUE,
+  TYPE_SPACE: TOKEN_TYPE_SPACE,
+  TYPE_NEW_LINE: TOKEN_TYPE_NEW_LINE,
+  convertTagToText,
+  getTagName,
+  getTokenColumn,
+  getTokenLine,
+  getTokenValue,
+  isAttrNameToken,
+  isAttrValueToken,
+  isTagStart,
+  isTagToken,
+  isTextToken,
+  isTagEnd,
+};