JavaScript Parsing and Compilation: Full Guide

// Source code:
function add(a, b) { return a + b; }
 
// Scanner output (token stream):
// Token          Type           Value
// ─────────────────────────────────────
// function       KEYWORD        "function"
// add            IDENTIFIER     "add"
// (              LPAREN         "("
// a              IDENTIFIER     "a"
// ,              COMMA          ","
// b              IDENTIFIER     "b"
// )              RPAREN         ")"
// {              LBRACE         "{"
// return         KEYWORD        "return"
// a              IDENTIFIER     "a"
// +              ADD            "+"
// b              IDENTIFIER     "b"
// ;              SEMICOLON      ";"
// }              RBRACE         "}"
 
// Building a simple tokenizer to understand the process
class Tokenizer {
  #source;
  #pos = 0;
  #tokens = [];
 
  static KEYWORDS = new Set([
    "function", "return", "if", "else", "for", "while",
    "const", "let", "var", "class", "new", "this",
  ]);
 
  constructor(source) {
    this.#source = source;
  }
 
  tokenize() {
    while (this.#pos < this.#source.length) {
      this.#skipWhitespace();
      if (this.#pos >= this.#source.length) break;
 
      const char = this.#source[this.#pos];
 
      if (this.#isAlpha(char)) {
        this.#readIdentifierOrKeyword();
      } else if (this.#isDigit(char)) {
        this.#readNumber();
      } else if (char === '"' || char === "'") {
        this.#readString(char);
      } else {
        this.#readOperatorOrPunctuation();
      }
    }
    return this.#tokens;
  }
 
  #readIdentifierOrKeyword() {
    let value = "";
    while (this.#pos < this.#source.length && this.#isAlphaNumeric(this.#source[this.#pos])) {
      value += this.#source[this.#pos++];
    }
    const type = Tokenizer.KEYWORDS.has(value) ? "KEYWORD" : "IDENTIFIER";
    this.#tokens.push({ type, value });
  }
 
  #readNumber() {
    let value = "";
    let isFloat = false;
    while (this.#pos < this.#source.length) {
      const c = this.#source[this.#pos];
      if (c === "." && !isFloat) { isFloat = true; value += c; this.#pos++; }
      else if (this.#isDigit(c)) { value += c; this.#pos++; }
      else break;
    }
    this.#tokens.push({ type: "NUMBER", value: parseFloat(value) });
  }
 
  #readString(quote) {
    this.#pos++; // skip opening quote
    let value = "";
    while (this.#pos < this.#source.length && this.#source[this.#pos] !== quote) {
      if (this.#source[this.#pos] === "\\") {
        this.#pos++;
        value += this.#source[this.#pos] || "";
      } else {
        value += this.#source[this.#pos];
      }
      this.#pos++;
    }
    this.#pos++; // skip closing quote
    this.#tokens.push({ type: "STRING", value });
  }
 
  #readOperatorOrPunctuation() {
    const char = this.#source[this.#pos++];
    const ops = { "+": "ADD", "-": "SUB", "*": "MUL", "/": "DIV",
                  "=": "ASSIGN", "(": "LPAREN", ")": "RPAREN",
                  "{": "LBRACE", "}": "RBRACE", ";": "SEMICOLON",
                  ",": "COMMA", ".": "DOT" };
    this.#tokens.push({ type: ops[char] || "UNKNOWN", value: char });
  }
 
  #skipWhitespace() {
    while (this.#pos < this.#source.length && /\s/.test(this.#source[this.#pos])) {
      this.#pos++;
    }
  }
 
  #isAlpha(c) { return /[a-zA-Z_$]/.test(c); }
  #isDigit(c) { return /[0-9]/.test(c); }
  #isAlphaNumeric(c) { return /[a-zA-Z0-9_$]/.test(c); }
}
 
const tokens = new Tokenizer("const x = 42 + y;").tokenize();
// [{type:"KEYWORD",value:"const"}, {type:"IDENTIFIER",value:"x"},
//  {type:"ASSIGN",value:"="}, {type:"NUMBER",value:42},
//  {type:"ADD",value:"+"}, {type:"IDENTIFIER",value:"y"},
//  {type:"SEMICOLON",value:";"}]

// V8 uses two parsing strategies to optimize startup time
 
// EAGER PARSING: Full AST built immediately
// Applied to: top-level code, IIFEs, and functions called during load
 
// Top-level statements are eagerly parsed
const config = { debug: true, port: 3000 };
console.log("App starting");
 
// IIFEs are eagerly parsed (V8 detects the wrapping parens)
const utils = (function () {
  // Full AST built immediately because V8 knows
  // this function will execute right away
  function formatDate(d) {
    return d.toISOString().split("T")[0];
  }
  return { formatDate };
})();
 
// LAZY PARSING (Pre-parsing): Only syntax validation, no AST
// Applied to: function declarations and expressions not called during load
 
function rarelyUsedFeature() {
  // Pre-parser scans this body but does NOT build an AST
  // It only checks: syntax is valid, finds variable declarations,
  // determines which variables are captured by inner functions
  const data = loadExpensiveData();
  return processData(data);
}
 
// When rarelyUsedFeature() is first called, V8 must re-parse
// the function body fully (this is the cost of lazy parsing)
 
// OPTIMIZATION: Avoid unnecessary function wrapping
// BAD: Forces lazy parse + re-parse of startup code
function initialize() {
  setupDOM();
  bindEvents();
  loadData();
}
initialize(); // Lazy parsed first, then re-parsed when called
 
// BETTER: Top-level code is eagerly parsed (no re-parse)
setupDOM();
bindEvents();
loadData();
 
// COMPILE HINTS: V8 respects some patterns
// Wrapping in parentheses hints at eager compilation
const module = (function () {
  // Eagerly parsed due to IIFE pattern
  return {};
})();
 
// The cost comparison:
// Lazy parse: ~50% of full parse cost (syntax check only)
// Full parse: 100% (AST construction + scope analysis)
// Re-parse:   100% (must do full parse when function is called)
// So lazy parsing saves time only if the function is never called

// During parsing, V8 performs scope analysis to determine:
// 1. Where each variable is declared
// 2. Whether variables are captured by closures
// 3. Whether the 'eval' keyword is used (disables optimizations)
 
// SCOPE CHAIN CONSTRUCTION
function outer() {
  const x = 10;       // Declared in 'outer' scope
 
  function middle() {
    const y = 20;     // Declared in 'middle' scope
 
    function inner() {
      const z = 30;   // Declared in 'inner' scope
      return x + y + z; // x and y are "captured" from outer scopes
    }
 
    return inner();
  }
 
  return middle();
}
 
// Scope analysis reveals:
// - 'x' is in outer scope, captured by inner (must be heap-allocated)
// - 'y' is in middle scope, captured by inner (must be heap-allocated)
// - 'z' is in inner scope, not captured (can stay on stack)
 
// CONTEXT ALLOCATION
// Variables captured by closures cannot live on the stack
// (the stack frame disappears when the function returns)
// So V8 allocates them in a "Context" object on the heap
 
function createCounter() {
  let count = 0;  // Captured by returned function -> heap allocated
 
  return {
    increment() { count++; },   // Captures 'count'
    getCount() { return count; }, // Captures 'count'
  };
}
// 'count' lives in a Context object shared by increment and getCount
 
// EVAL DISABLES OPTIMIZATIONS
function withEval(code) {
  const local = 42;
  eval(code); // V8 cannot determine what variables 'code' accesses
  // All local variables must be kept accessible (no optimization)
  return local;
}
// Because eval can access any variable, V8 must:
// - Heap-allocate ALL locals (not just captured ones)
// - Disable scope-based optimizations
// - Use slower dynamic variable lookup
 
// IMPACT: Avoid eval and 'with' for performance
// BAD
function slow(input) {
  const data = JSON.parse(input);
  eval("console.log(data)"); // Forces all locals to heap
}
 
// GOOD
function fast(input) {
  const data = JSON.parse(input);
  console.log(data); // Locals can stay on stack
}

// After parsing, Ignition compiles the AST to bytecode
// Bytecode is a compact, platform-independent instruction set
 
// Simple function and its bytecode
function greet(name) {
  return "Hello, " + name + "!";
}
 
// Ignition bytecode (simplified):
//   LdaConstant [0]         // Load constant "Hello, " into accumulator
//   Add a0, [0]             // Concatenate with parameter 'name'
//   AddSmi [1]              // Add constant "!" (from constant pool)
//   Return                  // Return accumulator
 
// More complex example with branches
function max(a, b) {
  if (a > b) {
    return a;
  }
  return b;
}
 
// Ignition bytecode:
//   Ldar a0                 // Load 'a' into accumulator
//   TestGreaterThan a1, [0] // Compare accumulator > 'b'
//   JumpIfFalse [3]         // Jump 3 bytes ahead if false
//   Ldar a0                 // Load 'a' (true branch)
//   Return                  // Return 'a'
//   Ldar a1                 // Load 'b' (false branch)
//   Return                  // Return 'b'
 
// Loop bytecode
function sumTo(n) {
  let total = 0;
  for (let i = 0; i <= n; i++) {
    total += i;
  }
  return total;
}
 
// Ignition bytecode:
//   LdaZero                 // total = 0
//   Star r0                 // Store in register r0
//   LdaZero                 // i = 0
//   Star r1                 // Store in register r1
// Loop:
//   Ldar r1                 // Load i
//   TestLessThanOrEqual a0  // i <= n?
//   JumpIfFalse [End]       // Exit if false
//   Ldar r0                 // Load total
//   Add r1, [0]             // total + i
//   Star r0                 // Store result in total
//   Ldar r1                 // Load i
//   Inc [1]                 // i++
//   Star r1                 // Store i
//   Jump [Loop]             // Back to loop start
// End:
//   Ldar r0                 // Load total
//   Return                  // Return total
 
// VIEWING REAL BYTECODE
// In Node.js, use: node --print-bytecode --print-bytecode-filter=functionName
// Example: node --print-bytecode --print-bytecode-filter=sumTo script.js

// Code patterns that affect parsing and compilation speed
 
// 1. SCRIPT STREAMING: V8 can parse while downloading
// Large script files benefit from streaming parse
// The parser works on chunks as they arrive over the network
 
// 2. CODE SPLITTING reduces initial parse cost
// BAD: One large bundle forces parsing of everything
// <script src="bundle.js"></script>  // 5MB - all parsed upfront
 
// GOOD: Split into route-based chunks
// <script src="core.js"></script>     // 200KB - parsed immediately
// <script src="dashboard.js" async></script> // Parsed when needed
 
// 3. MINIMIZE NESTED FUNCTIONS for better lazy parsing
// BAD: Deeply nested functions have compounding re-parse cost
function level1() {
  function level2() {
    function level3() {
      function level4() {
        return "deep";
      }
      return level4();
    }
    return level3();
  }
  return level2();
}
// Each level triggers a full re-parse cascade
 
// BETTER: Flat function structure
function handler4() { return "deep"; }
function handler3() { return handler4(); }
function handler2() { return handler3(); }
function handler1() { return handler2(); }
 
// 4. BYTECODE CACHING: V8 caches compiled bytecode
// First visit: parse + compile (slow)
// Subsequent visits: load cached bytecode (fast)
// Cache key: script URL + source hash
// To benefit: use versioned URLs (app.v2.js) and avoid inline scripts
 
// 5. AVOID LARGE SWITCH STATEMENTS
// V8 handles switch with many cases less efficiently
// BAD: Large switch with 100+ cases
function handleAction(type) {
  switch (type) {
    case "A": return doA();
    case "B": return doB();
    // ... 100 more cases
  }
}
 
// BETTER: Object lookup
const handlers = {
  A: doA,
  B: doB,
  // ... handlers added dynamically
};
 
function handleAction(type) {
  return handlers[type]?.() ?? handleDefault();
}