Add tokens to js parsers (#8529)

src/language-js/clean.js

@@ -14,6 +14,7 @@ function clean(ast, newObj, parent) {
     "loc",
     "flags",
     "errors",
+    "tokens",
   ].forEach((name) => {
     delete newObj[name];
   });

src/language-js/parser-babel.js

@@ -37,6 +37,7 @@ function babelOptions({ sourceType, extraPlugins = [] }) {
       ["recordAndTuple", { syntaxType: "hash" }],
       ...extraPlugins,
     ],
+    tokens: true,
   };
 }
 
@@ -97,7 +98,7 @@ function createParse(parseMethod, ...pluginCombinations) {
           : { start: { line: 0, column: 0 } }
       );
     }
-    delete ast.tokens;
+
     return postprocess(ast, { ...opts, originalText: text });
   };
 }

src/language-js/parser-flow.js

@@ -17,6 +17,7 @@ function parse(text, parsers, opts) {
     esproposal_export_star_as: true,
     esproposal_optional_chaining: true,
     esproposal_nullish_coalescing: true,
+    tokens: true,
   });
 
   if (ast.errors.length > 0) {

src/language-js/parser-typescript.js

@@ -39,6 +39,7 @@ function tryParseTypeScript(text, jsx) {
     comment: true,
     useJSXTextNode: true,
     jsx,
+    tokens: true,
   });
 }
 

src/main/comments.js

@@ -58,7 +58,9 @@ function getSortedChildNodes(node, options, resultArray) {
           (n) =>
             n !== "enclosingNode" &&
             n !== "precedingNode" &&
-            n !== "followingNode"
+            n !== "followingNode" &&
+            n !== "tokens" &&
+            n !== "comments"
         )
         .map((n) => node[n]));
 

src/main/core.js

@@ -33,8 +33,8 @@ function attachComments(text, ast, opts) {
     delete ast.comments;
     comments.attach(astComments, ast, text, opts);
   }
-  ast.tokens = [];
   opts[Symbol.for("comments")] = astComments || [];
+  opts[Symbol.for("tokens")] = ast.tokens || [];
   opts.originalText = opts.parser === "yaml" ? text : text.trimEnd();
   return astComments;
 }

src/main/multiparser.js

@@ -41,6 +41,7 @@ function textToDoc(text, partialNextOptions, parentOptions, printAstToDoc) {
   delete ast.comments;
   comments.attach(astComments, ast, text, nextOptions);
   nextOptions[Symbol.for("comments")] = astComments || [];
+  nextOptions[Symbol.for("tokens")] = ast.tokens || [];
   const doc = printAstToDoc(ast, nextOptions);
   comments.ensureAllCommentsPrinted(astComments);
   return doc;