summaryrefslogtreecommitdiffstats
path: root/node_modules/@webassemblyjs/wast-parser/esm/tokenizer.js
diff options
context:
space:
mode:
authorGravatar Piotr Russ <mail@pruss.it> 2020-11-16 00:10:28 +0100
committerGravatar Piotr Russ <mail@pruss.it> 2020-11-16 00:10:28 +0100
commite06ec920f7a5d784e674c4c4b4e6d1da3dc7391d (patch)
tree55713f725f77b44ebfec86e4eec3ce33e71458ca /node_modules/@webassemblyjs/wast-parser/esm/tokenizer.js
downloadwebsite_creator-e06ec920f7a5d784e674c4c4b4e6d1da3dc7391d.tar.gz
website_creator-e06ec920f7a5d784e674c4c4b4e6d1da3dc7391d.tar.bz2
website_creator-e06ec920f7a5d784e674c4c4b4e6d1da3dc7391d.zip
api, login, auth
Diffstat (limited to 'node_modules/@webassemblyjs/wast-parser/esm/tokenizer.js')
-rw-r--r--node_modules/@webassemblyjs/wast-parser/esm/tokenizer.js434
1 files changed, 434 insertions, 0 deletions
diff --git a/node_modules/@webassemblyjs/wast-parser/esm/tokenizer.js b/node_modules/@webassemblyjs/wast-parser/esm/tokenizer.js
new file mode 100644
index 0000000..30f9098
--- /dev/null
+++ b/node_modules/@webassemblyjs/wast-parser/esm/tokenizer.js
@@ -0,0 +1,434 @@
+import { FSM, makeTransition } from "@webassemblyjs/helper-fsm";
+import { codeFrameFromSource } from "@webassemblyjs/helper-code-frame";
+
+// eslint-disable-next-line
+function getCodeFrame(source, line, column) {
+ var loc = {
+ start: {
+ line: line,
+ column: column
+ }
+ };
+ return "\n" + codeFrameFromSource(source, loc) + "\n";
+}
+
+var WHITESPACE = /\s/;
+var PARENS = /\(|\)/;
+var LETTERS = /[a-z0-9_/]/i;
+var idchar = /[a-z0-9!#$%&*+./:<=>?@\\[\]^_`|~-]/i;
+var valtypes = ["i32", "i64", "f32", "f64"];
+var NUMBERS = /[0-9|.|_]/;
+var NUMBER_KEYWORDS = /nan|inf/;
+
+function isNewLine(char) {
+ return char.charCodeAt(0) === 10 || char.charCodeAt(0) === 13;
+}
+
+function Token(type, value, start, end) {
+ var opts = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : {};
+ var token = {
+ type: type,
+ value: value,
+ loc: {
+ start: start,
+ end: end
+ }
+ };
+
+ if (Object.keys(opts).length > 0) {
+ // $FlowIgnore
+ token["opts"] = opts;
+ }
+
+ return token;
+}
+
+var tokenTypes = {
+ openParen: "openParen",
+ closeParen: "closeParen",
+ number: "number",
+ string: "string",
+ name: "name",
+ identifier: "identifier",
+ valtype: "valtype",
+ dot: "dot",
+ comment: "comment",
+ equal: "equal",
+ keyword: "keyword"
+};
+export var keywords = {
+ module: "module",
+ func: "func",
+ param: "param",
+ result: "result",
+ export: "export",
+ loop: "loop",
+ block: "block",
+ if: "if",
+ then: "then",
+ else: "else",
+ call: "call",
+ call_indirect: "call_indirect",
+ import: "import",
+ memory: "memory",
+ table: "table",
+ global: "global",
+ anyfunc: "anyfunc",
+ mut: "mut",
+ data: "data",
+ type: "type",
+ elem: "elem",
+ start: "start",
+ offset: "offset"
+};
+var NUMERIC_SEPARATOR = "_";
+/**
+ * Build the FSM for number literals
+ */
+
+var numberLiteralFSM = new FSM({
+ START: [makeTransition(/-|\+/, "AFTER_SIGN"), makeTransition(/nan:0x/, "NAN_HEX", {
+ n: 6
+ }), makeTransition(/nan|inf/, "STOP", {
+ n: 3
+ }), makeTransition(/0x/, "HEX", {
+ n: 2
+ }), makeTransition(/[0-9]/, "DEC"), makeTransition(/\./, "DEC_FRAC")],
+ AFTER_SIGN: [makeTransition(/nan:0x/, "NAN_HEX", {
+ n: 6
+ }), makeTransition(/nan|inf/, "STOP", {
+ n: 3
+ }), makeTransition(/0x/, "HEX", {
+ n: 2
+ }), makeTransition(/[0-9]/, "DEC"), makeTransition(/\./, "DEC_FRAC")],
+ DEC_FRAC: [makeTransition(/[0-9]/, "DEC_FRAC", {
+ allowedSeparator: NUMERIC_SEPARATOR
+ }), makeTransition(/e|E/, "DEC_SIGNED_EXP")],
+ DEC: [makeTransition(/[0-9]/, "DEC", {
+ allowedSeparator: NUMERIC_SEPARATOR
+ }), makeTransition(/\./, "DEC_FRAC"), makeTransition(/e|E/, "DEC_SIGNED_EXP")],
+ DEC_SIGNED_EXP: [makeTransition(/\+|-/, "DEC_EXP"), makeTransition(/[0-9]/, "DEC_EXP")],
+ DEC_EXP: [makeTransition(/[0-9]/, "DEC_EXP", {
+ allowedSeparator: NUMERIC_SEPARATOR
+ })],
+ HEX: [makeTransition(/[0-9|A-F|a-f]/, "HEX", {
+ allowedSeparator: NUMERIC_SEPARATOR
+ }), makeTransition(/\./, "HEX_FRAC"), makeTransition(/p|P/, "HEX_SIGNED_EXP")],
+ HEX_FRAC: [makeTransition(/[0-9|A-F|a-f]/, "HEX_FRAC", {
+ allowedSeparator: NUMERIC_SEPARATOR
+ }), makeTransition(/p|P|/, "HEX_SIGNED_EXP")],
+ HEX_SIGNED_EXP: [makeTransition(/[0-9|+|-]/, "HEX_EXP")],
+ HEX_EXP: [makeTransition(/[0-9]/, "HEX_EXP", {
+ allowedSeparator: NUMERIC_SEPARATOR
+ })],
+ NAN_HEX: [makeTransition(/[0-9|A-F|a-f]/, "NAN_HEX", {
+ allowedSeparator: NUMERIC_SEPARATOR
+ })],
+ STOP: []
+}, "START", "STOP");
+export function tokenize(input) {
+ var current = 0;
+ var char = input[current]; // Used by SourceLocation
+
+ var column = 1;
+ var line = 1;
+ var tokens = [];
+ /**
+ * Creates a pushToken function for a given type
+ */
+
+ function pushToken(type) {
+ return function (v) {
+ var opts = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
+ var startColumn = opts.startColumn || column - String(v).length;
+ delete opts.startColumn;
+ var endColumn = opts.endColumn || startColumn + String(v).length - 1;
+ delete opts.endColumn;
+ var start = {
+ line: line,
+ column: startColumn
+ };
+ var end = {
+ line: line,
+ column: endColumn
+ };
+ tokens.push(Token(type, v, start, end, opts));
+ };
+ }
+ /**
+ * Functions to save newly encountered tokens
+ */
+
+
+ var pushCloseParenToken = pushToken(tokenTypes.closeParen);
+ var pushOpenParenToken = pushToken(tokenTypes.openParen);
+ var pushNumberToken = pushToken(tokenTypes.number);
+ var pushValtypeToken = pushToken(tokenTypes.valtype);
+ var pushNameToken = pushToken(tokenTypes.name);
+ var pushIdentifierToken = pushToken(tokenTypes.identifier);
+ var pushKeywordToken = pushToken(tokenTypes.keyword);
+ var pushDotToken = pushToken(tokenTypes.dot);
+ var pushStringToken = pushToken(tokenTypes.string);
+ var pushCommentToken = pushToken(tokenTypes.comment);
+ var pushEqualToken = pushToken(tokenTypes.equal);
+ /**
+ * Can be used to look at the next character(s).
+ *
+ * The default behavior `lookahead()` simply returns the next character without consuming it.
+ * Letters are always returned in lowercase.
+ *
+ * @param {number} length How many characters to query. Default = 1
+ * @param {number} offset How many characters to skip forward from current one. Default = 1
+ *
+ */
+
+ function lookahead() {
+ var length = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 1;
+ var offset = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 1;
+ return input.substring(current + offset, current + offset + length).toLowerCase();
+ }
+ /**
+ * Advances the cursor in the input by a certain amount
+ *
+ * @param {number} amount How many characters to consume. Default = 1
+ */
+
+
+ function eatCharacter() {
+ var amount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 1;
+ column += amount;
+ current += amount;
+ char = input[current];
+ }
+
+ while (current < input.length) {
+ // ;;
+ if (char === ";" && lookahead() === ";") {
+ var startColumn = column;
+ eatCharacter(2);
+ var text = "";
+
+ while (!isNewLine(char)) {
+ text += char;
+ eatCharacter();
+
+ if (char === undefined) {
+ break;
+ }
+ }
+
+ var endColumn = column;
+ pushCommentToken(text, {
+ type: "leading",
+ startColumn: startColumn,
+ endColumn: endColumn
+ });
+ continue;
+ } // (;
+
+
+ if (char === "(" && lookahead() === ";") {
+ var _startColumn = column;
+ eatCharacter(2);
+ var _text = ""; // ;)
+
+ while (true) {
+ char = input[current];
+
+ if (char === ";" && lookahead() === ")") {
+ eatCharacter(2);
+ break;
+ }
+
+ _text += char;
+ eatCharacter();
+
+ if (isNewLine(char)) {
+ line++;
+ column = 0;
+ }
+ }
+
+ var _endColumn = column;
+ pushCommentToken(_text, {
+ type: "block",
+ startColumn: _startColumn,
+ endColumn: _endColumn
+ });
+ continue;
+ }
+
+ if (char === "(") {
+ pushOpenParenToken(char);
+ eatCharacter();
+ continue;
+ }
+
+ if (char === "=") {
+ pushEqualToken(char);
+ eatCharacter();
+ continue;
+ }
+
+ if (char === ")") {
+ pushCloseParenToken(char);
+ eatCharacter();
+ continue;
+ }
+
+ if (isNewLine(char)) {
+ line++;
+ eatCharacter();
+ column = 0;
+ continue;
+ }
+
+ if (WHITESPACE.test(char)) {
+ eatCharacter();
+ continue;
+ }
+
+ if (char === "$") {
+ var _startColumn2 = column;
+ eatCharacter();
+ var value = "";
+
+ while (idchar.test(char)) {
+ value += char;
+ eatCharacter();
+ }
+
+ var _endColumn2 = column;
+ pushIdentifierToken(value, {
+ startColumn: _startColumn2,
+ endColumn: _endColumn2
+ });
+ continue;
+ }
+
+ if (NUMBERS.test(char) || NUMBER_KEYWORDS.test(lookahead(3, 0)) || char === "-" || char === "+") {
+ var _startColumn3 = column;
+
+ var _value = numberLiteralFSM.run(input.slice(current));
+
+ if (_value === "") {
+ throw new Error(getCodeFrame(input, line, column) + "Unexpected character " + JSON.stringify(char));
+ }
+
+ pushNumberToken(_value, {
+ startColumn: _startColumn3
+ });
+ eatCharacter(_value.length);
+
+ if (char && !PARENS.test(char) && !WHITESPACE.test(char)) {
+ throw new Error(getCodeFrame(input, line, column) + "Unexpected character " + JSON.stringify(char));
+ }
+
+ continue;
+ }
+
+ if (char === '"') {
+ var _startColumn4 = column;
+ var _value2 = "";
+ eatCharacter(); // "
+
+ while (char !== '"') {
+ if (isNewLine(char)) {
+ throw new Error(getCodeFrame(input, line, column) + "Unexpected character " + JSON.stringify(char));
+ }
+
+ _value2 += char;
+ eatCharacter(); // char
+ }
+
+ eatCharacter(); // "
+
+ var _endColumn3 = column;
+ pushStringToken(_value2, {
+ startColumn: _startColumn4,
+ endColumn: _endColumn3
+ });
+ continue;
+ }
+
+ if (LETTERS.test(char)) {
+ var _value3 = "";
+ var _startColumn5 = column;
+
+ while (char && LETTERS.test(char)) {
+ _value3 += char;
+ eatCharacter();
+ }
+ /*
+ * Handle MemberAccess
+ */
+
+
+ if (char === ".") {
+ var dotStartColumn = column;
+
+ if (valtypes.indexOf(_value3) !== -1) {
+ pushValtypeToken(_value3, {
+ startColumn: _startColumn5
+ });
+ } else {
+ pushNameToken(_value3);
+ }
+
+ eatCharacter();
+ _value3 = "";
+ var nameStartColumn = column;
+
+ while (LETTERS.test(char)) {
+ _value3 += char;
+ eatCharacter();
+ }
+
+ pushDotToken(".", {
+ startColumn: dotStartColumn
+ });
+ pushNameToken(_value3, {
+ startColumn: nameStartColumn
+ });
+ continue;
+ }
+ /*
+ * Handle keywords
+ */
+ // $FlowIgnore
+
+
+ if (typeof keywords[_value3] === "string") {
+ pushKeywordToken(_value3, {
+ startColumn: _startColumn5
+ });
+ continue;
+ }
+ /*
+ * Handle types
+ */
+
+
+ if (valtypes.indexOf(_value3) !== -1) {
+ pushValtypeToken(_value3, {
+ startColumn: _startColumn5
+ });
+ continue;
+ }
+ /*
+ * Handle literals
+ */
+
+
+ pushNameToken(_value3, {
+ startColumn: _startColumn5
+ });
+ continue;
+ }
+
+ throw new Error(getCodeFrame(input, line, column) + "Unexpected character " + JSON.stringify(char));
+ }
+
+ return tokens;
+}
+export var tokens = tokenTypes; \ No newline at end of file