Merge pull request #1500 from bilawalsidhu/sameh/vue-loader-cache

Cache compiled components and load them on the test thread

build/registerVue.js

@@ -1,5 +1,8 @@
-import { register } from 'node:module';
+import { registerHooks } from 'node:module';
+import { load } from './vueLoader.js';
 
 // Lets Node import single-file components: `node --import=<this file>`.
-// Hosts reach it as `<core package>/build/register-vue`.
-register('./vueLoader.js', import.meta.url);
+// Hosts reach it as `<core package>/build/register-vue`. The hooks run on the
+// loading thread: off-thread hooks (module.register) would add a round trip
+// to every module a test imports, not only the components.
+registerHooks({ load });

build/vueLoader.js

@@ -1,14 +1,32 @@
-import { readFile } from 'node:fs/promises';
+import { createHash, randomBytes } from 'node:crypto';
+import {
+  lstatSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { createRequire } from 'node:module';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { compileScript, compileTemplate, parse } from 'vue/compiler-sfc';
 import { VUE_TEMPLATE_OPTIONS } from './vueOptions.js';
 
+const require = createRequire(import.meta.url);
+
+// The compiler loads on the first component that is not cached, so a test
+// process that imports no views, or only cached ones, never pays for it.
+let compiler = null;
+const sfcCompiler = () => (compiler ??= require('vue/compiler-sfc'));
+
 /**
  * Compile a single-file component to an ES module for Node, so unit tests (the
  * application's and a host's) can import components without a bundler.
  * Styles are dropped: tests read the global stylesheet separately.
  */
 export function compileVueModule(source, filename) {
+  const { compileScript, compileTemplate, parse } = sfcCompiler();
   // The template is parsed with the SFC; parse it as the browser build does.
   const { descriptor, errors } = parse(source, {
     filename,
@@ -40,14 +58,113 @@ export function compileVueModule(source, filename) {
   return `${script}\n${template}\nif (typeof render === 'function') __sfc__.render = render;\nexport default __sfc__;\n`;
 }
 
-/** Node module hook: load `.vue` files as compiled modules. */
-export async function load(url, context, nextLoad) {
+// The loader's own code is part of every key, so a change to how components
+// compile never reuses output from before it.
+let loaderFingerprint = null;
+const fingerprint = () =>
+  (loaderFingerprint ??= createHash('sha256')
+    .update(readFileSync(fileURLToPath(import.meta.url)))
+    .update(readFileSync(new URL('./vueOptions.js', import.meta.url)))
+    .update(require('vue/package.json').version)
+    .digest('hex'));
+
+/**
+ * Where compiled components are kept between test processes, never a
+ * package's own tree (a host's checks refuse files inside Core): the system
+ * temp directory, or on Windows the per-user local app data directory, since
+ * a temp directory there may be shared and its ownership is not checked.
+ * GEV_VUE_CACHE=0 compiles every time.
+ */
+export function vueCacheDirectory(
+  env = process.env,
+  platform = process.platform,
+) {
+  if (env.GEV_VUE_CACHE === '0') return null;
+  if (platform === 'win32') {
+    return env.LOCALAPPDATA ? path.join(env.LOCALAPPDATA, 'gev-vue-sfc') : null;
+  }
+  return path.join(tmpdir(), `gev-vue-sfc-${process.getuid()}`);
+}
+
+/** The cache entry for one component: its path, its text and the loader. */
+export function vueCacheKey(source, filename) {
+  return createHash('sha256')
+    .update(JSON.stringify([fingerprint(), filename]))
+    .update('\0')
+    .update(source)
+    .digest('hex');
+}
+
+/**
+ * Compile a component, reusing a previous process's result for the same text.
+ * A cache that cannot be read or written only costs the compile.
+ */
+export function cachedVueModule(
+  source,
+  filename,
+  directory = vueCacheDirectory(),
+) {
+  if (!directory || !ownCacheDirectory(directory)) {
+    return compileVueModule(source, filename);
+  }
+  const entry = path.join(directory, `${vueCacheKey(source, filename)}.mjs`);
+  try {
+    return readFileSync(entry, 'utf8');
+  } catch {
+    // Not compiled yet.
+  }
+  const compiled = compileVueModule(source, filename);
+  // Concurrent test processes and threads may compile the same component;
+  // each writes its own new file and the rename makes the entry appear whole.
+  const partial = `${entry}.${randomBytes(6).toString('hex')}.tmp`;
+  try {
+    writeFileSync(partial, compiled, { flag: 'wx', mode: 0o600 });
+    renameSync(partial, entry);
+  } catch {
+    // An unwritable directory, or a rename another process holds open,
+    // leaves the cache without this entry.
+    try {
+      rmSync(partial, { force: true });
+    } catch {
+      // A partial file that cannot be removed is only clutter.
+    }
+  }
+  return compiled;
+}
+
+/**
+ * Cached modules run as code, so the directory must be this user's own: a
+ * real directory (not a link), owned by the current user and writable by no
+ * one else. In a shared temp directory another user could otherwise create it
+ * first and plant modules. On Windows it sits in the user's own profile.
+ */
+const checkedDirectories = new Map();
+export function ownCacheDirectory(directory) {
+  if (!checkedDirectories.has(directory)) {
+    checkedDirectories.set(directory, isOwnDirectory(directory));
+  }
+  return checkedDirectories.get(directory);
+}
+
+function isOwnDirectory(directory) {
+  try {
+    mkdirSync(directory, { recursive: true, mode: 0o700 });
+    const stats = lstatSync(directory);
+    if (!stats.isDirectory()) return false;
+    if (process.platform === 'win32') return true;
+    return stats.uid === process.getuid() && (stats.mode & 0o022) === 0;
+  } catch {
+    return false;
+  }
+}
+
+/** Synchronous module hook (module.registerHooks): `.vue` loads compiled. */
+export function load(url, context, nextLoad) {
   if (!url.endsWith('.vue')) return nextLoad(url, context);
   const filename = fileURLToPath(url);
-  const source = await readFile(filename, 'utf8');
   return {
     format: 'module',
-    source: compileVueModule(source, filename),
+    source: cachedVueModule(readFileSync(filename, 'utf8'), filename),
     shortCircuit: true,
   };
 }

changelog.d/1500-vue-loader-cache.md

@@ -0,0 +1,4 @@
+- `build/register-vue` loads components on the test thread and keeps compiled
+  components between test processes in a per-user cache directory, so Node
+  tests that import views run about twice as fast. `GEV_VUE_CACHE=0` compiles
+  every time (samehkhamis, #1500).

scripts/format-scope.json

@@ -68,6 +68,7 @@
   "server/standalone/vite.config.js",
   "vite.config.js",
   "src/tooling/viteBuild.test.mjs",
+  "src/tooling/vueLoader.test.mjs",
   "src/tooling/allowedHosts.test.mjs",
   "server/providers/common/http.js",
   "server/providers/common/query.js",

src/tooling/vueLoader.test.mjs

@@ -0,0 +1,111 @@
+import assert from 'node:assert/strict';
+import {
+  chmodSync,
+  mkdirSync,
+  mkdtempSync,
+  readdirSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+import {
+  cachedVueModule,
+  compileVueModule,
+  ownCacheDirectory,
+  vueCacheDirectory,
+  vueCacheKey,
+} from '../../build/vueLoader.js';
+
+const SOURCE =
+  '<script setup>\nconst label = "Hi";\n</script>\n<template><b>{{ label }}</b></template>\n';
+const FILE = '/virtual/Greeting.vue';
+
+test('a component compiles once and later processes read the cached module', (t) => {
+  const directory = mkdtempSync(path.join(tmpdir(), 'gev-vue-cache-test-'));
+  t.after(() => rmSync(directory, { recursive: true, force: true }));
+  const compiled = cachedVueModule(SOURCE, FILE, directory);
+  assert.equal(compiled, compileVueModule(SOURCE, FILE));
+  const entries = readdirSync(directory);
+  assert.deepEqual(entries, [`${vueCacheKey(SOURCE, FILE)}.mjs`]);
+  // A cached entry is returned as stored, without compiling again.
+  writeFileSync(path.join(directory, entries[0]), 'export default "cached";');
+  assert.equal(
+    cachedVueModule(SOURCE, FILE, directory),
+    'export default "cached";',
+  );
+});
+
+test('an edited component or another path is a new cache entry', () => {
+  const key = vueCacheKey(SOURCE, FILE);
+  assert.notEqual(vueCacheKey(`${SOURCE}\n`, FILE), key);
+  assert.notEqual(vueCacheKey(SOURCE, '/virtual/Other.vue'), key);
+  assert.equal(vueCacheKey(SOURCE, FILE), key);
+});
+
+test('GEV_VUE_CACHE=0 compiles every time; otherwise the cache sits in a per-user directory', () => {
+  assert.equal(vueCacheDirectory({ GEV_VUE_CACHE: '0' }), null);
+  if (process.platform !== 'win32') {
+    const directory = vueCacheDirectory({});
+    assert.equal(path.dirname(directory), tmpdir());
+    assert.match(path.basename(directory), /^gev-vue-sfc-/);
+  }
+  // Windows temp directories may be shared, so the cache lives in the
+  // user's profile, or not at all.
+  assert.equal(
+    vueCacheDirectory(
+      { LOCALAPPDATA: 'C:\\Users\\a\\AppData\\Local' },
+      'win32',
+    ),
+    path.join('C:\\Users\\a\\AppData\\Local', 'gev-vue-sfc'),
+  );
+  assert.equal(vueCacheDirectory({}, 'win32'), null);
+  assert.equal(
+    cachedVueModule(SOURCE, FILE, null),
+    compileVueModule(SOURCE, FILE),
+  );
+});
+
+test('a cache directory that cannot be written still returns the compiled module', (t) => {
+  const directory = mkdtempSync(path.join(tmpdir(), 'gev-vue-cache-test-'));
+  t.after(() => rmSync(directory, { recursive: true, force: true }));
+  // A file where the cache directory should be makes every write fail.
+  const blocked = path.join(directory, 'blocked');
+  writeFileSync(blocked, '');
+  assert.equal(
+    cachedVueModule(SOURCE, FILE, blocked),
+    compileVueModule(SOURCE, FILE),
+  );
+});
+
+test(
+  'a cache directory others can write to, or a link to one, is never read',
+  {
+    skip: process.platform === 'win32',
+  },
+  (t) => {
+    const root = mkdtempSync(path.join(tmpdir(), 'gev-vue-cache-test-'));
+    t.after(() => rmSync(root, { recursive: true, force: true }));
+    const shared = path.join(root, 'shared');
+    mkdirSync(shared);
+    chmodSync(shared, 0o777);
+    // A module planted where the cache would look must not run.
+    writeFileSync(
+      path.join(shared, `${vueCacheKey(SOURCE, FILE)}.mjs`),
+      'export default "planted";',
+    );
+    assert.equal(ownCacheDirectory(shared), false);
+    assert.equal(
+      cachedVueModule(SOURCE, FILE, shared),
+      compileVueModule(SOURCE, FILE),
+    );
+    const own = path.join(root, 'own');
+    mkdirSync(own, { mode: 0o700 });
+    const link = path.join(root, 'link');
+    symlinkSync(own, link);
+    assert.equal(ownCacheDirectory(link), false);
+    assert.equal(ownCacheDirectory(own), true);
+  },
+);