tungsten 0.4.8 → 0.4.10

package/dist/bin/commands/devmon.js

@@ -1,52 +1,48 @@
-// Master process for `tungsten dev`: runs it as a child process and watches the same
-// build-notification marker file as warnOnFrameworkChange.ts (dist/.build-complete in the
-// framework package). When a framework rebuild touches it, kills and respawns the dev child
-// instead of just warning that a restart is needed.
+// Supervisor for `tungsten dev`: runs it as a child and watches the same build-notification marker
+// file as warnOnFrameworkChange.ts (dist/.build-complete in the framework package). Where that just
+// warns you to restart by hand, this kills the dev child on a framework rebuild and respawns it.
 import { spawn } from 'node:child_process';
 import { join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import chokidar from 'chokidar';
 import { cliError } from "../../cliOutput.js";
 import { getFrameworkDir } from "../../getFrameworkDir.js";
-const binIndexPath = fileURLToPath(new URL('../index.js', import.meta.url));
+import { cliArgs } from "../args.js";
+// devmon reads none of its arguments itself - it passes them straight through to the dev child
+cliArgs({}, {
+  handledElsewhere: ['--no-boot', '--profile', '--profile-modules']
+});
 const initialArgs = process.argv.slice(3);
-// Restarts always force --no-boot (no reopening the browser tab each time), but otherwise
-// keep whatever else devmon itself was passed - e.g. --profile.
-const restartArgs = [...initialArgs.filter(arg => arg !== '--no-boot'), '--no-boot'];
+// a restart must not reopen the browser tab, so it always carries --no-boot
+const restartArgs = initialArgs.includes('--no-boot') ? initialArgs : [...initialArgs, '--no-boot'];
+const binIndexPath = fileURLToPath(new URL('../index.js', import.meta.url));
 const buildMarkerPath = join(getFrameworkDir(), 'dist', '.build-complete');
 let child = null;
-let restarting = false;
-function startDev(args) {
+// tells the child's exit handler that we asked for it, rather than the run being over
+let restartWanted = false;
+function startDev(devArgs) {
   // __TUNGSTEN_DEVMON tells dev.ts to skip warnOnFrameworkChange() - devmon is already
   // restarting it, so that warning would just be confusing noise here.
-  child = spawn(process.execPath, [binIndexPath, 'dev', ...args], {
+  child = spawn(process.execPath, [binIndexPath, 'dev', ...devArgs], {
     stdio: 'inherit',
     env: {
       ...process.env,
       __TUNGSTEN_DEVMON: 'true'
     }
   });
-  child.on('exit', code => {
+  child.once('exit', code => {
     child = null;
-    // not one of our own restarts, so the dev process ended on its own (crash, or the
-    // user quit it directly) - nothing left for devmon to supervise
-    if (!restarting) process.exit(code ?? 0);
+    // nobody asked for this, so dev ended on its own - it crashed, or you quit it
+    if (!restartWanted) process.exit(code ?? 0);
+    restartWanted = false;
+    startDev(restartArgs);
   });
 }
-function restartDev() {
-  if (restarting) return;
-  restarting = true;
+function restartOnRebuild() {
+  if (restartWanted) return; // one is already on its way down
+  restartWanted = true;
   cliError('tungsten framework rebuilt - restarting tungsten dev');
-  if (child === null) {
-    restarting = false;
-    startDev(restartArgs);
-    return;
-  }
-  child.once('exit', () => {
-    restarting = false;
-    startDev(restartArgs);
-  });
-  child.kill();
+  child?.kill();
 }
 startDev(initialArgs);
 const watcher = chokidar.watch(buildMarkerPath, {
@@ -56,10 +52,10 @@ const watcher = chokidar.watch(buildMarkerPath, {
     pollInterval: 20
   }
 });
-watcher.on('all', restartDev);
+watcher.on('all', restartOnRebuild);
 function shutdown() {
   void watcher.close();
-  if (child !== null) child.kill();
+  child?.kill();
   process.exit(0);
 }
 process.on('SIGINT', shutdown);

package/dist/bin/commands/help.js

@@ -3,9 +3,9 @@ const command = process.argv[2];
 cliOut(`Usage: tungsten <command> [options]
 
 Commands:
-  dev                   Start dev mode [--no-boot] [--profile]
+  dev                   Start dev mode [--no-boot] [--profile] [--profile-modules]
   devmon                Dev mode, auto-restarts on framework rebuilds (same options as dev)
-  build                 Production build [--profile] [--quiet]
+  build                 Production build [--profile] [--profile-modules] [--quiet]
   start                 Start production server
   static                Preview a static site build
   clean                 Delete build assets

package/dist/bin/commands/start.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"start.d.ts","sourceRoot":"","sources":["../../../src/bin/commands/start.ts"],"names":[],"mappings":"AAgCA,OAAO,EAAE,CAAA"}
+{"version":3,"file":"start.d.ts","sourceRoot":"","sources":["../../../src/bin/commands/start.ts"],"names":[],"mappings":"AAkCA,OAAO,EAAE,CAAA"}

package/dist/bin/commands/start.js

@@ -1,7 +1,8 @@
 import { Framework } from "../../Framework.js";
+import { cliArgs } from "../args.js";
 //import dotenv from 'dotenv'
 //dotenv.config({ path: '.private.env' })
-console.log('tungsten start');
+cliArgs({}); // accepts nothing but --help
 //// prep the framework
 const framework = new Framework();
 await framework.init();

package/dist/bin/commands/static.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"static.d.ts","sourceRoot":"","sources":["../../../src/bin/commands/static.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,CAAA"}
+{"version":3,"file":"static.d.ts","sourceRoot":"","sources":["../../../src/bin/commands/static.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,CAAA"}

package/dist/bin/commands/static.js

@@ -1,6 +1,8 @@
 import open from 'open';
 import { getUserConfig } from "../../getUserConfig.js";
 import { StaticSiteServer } from "../../Servers/StaticSiteServer.js";
+import { cliArgs } from "../args.js";
+cliArgs({}); // accepts nothing but --help
 const staticServer = new StaticSiteServer(await getUserConfig());
 const staticServerLocation = await staticServer.start();
 open(staticServerLocation);

package/dist/bin/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/bin/index.ts"],"names":[],"mappings":"AAuEA,OAAO,EAAE,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/bin/index.ts"],"names":[],"mappings":"AA6GA,OAAO,EAAE,CAAA"}

package/dist/bin/index.js

@@ -1,7 +1,29 @@
+import { enableCompileCache, flushCompileCache } from 'node:module';
 import { cliOut, cliWarn } from "../cliOutput.js";
+import { beginStartupProfiling } from "../timeProfiler.js";
 // get command line args
 const args = process.argv.slice(2);
 const command = args[0];
+const profile = args.includes('--profile');
+// V8 bytecode cache for the command's import tree, which is compiled on the dynamic imports below.
+// Measured at ~14% off importing Framework.js once warm; the first run after a miss is ~8% slower
+// while it populates, so a single run straight after a rebuild looks like a regression.
+// NODE_DISABLE_COMPILE_CACHE=1 turns it off (NODE_COMPILE_CACHE does not - it names the cache
+// directory, so a value of 0 makes a folder called 0).
+const compileCache = enableCompileCache();
+// reported under --profile because "is it even on?" is otherwise only answerable by inspecting
+// the cache directory's timestamps - status 1 is enabled, 2 already enabled, 3 disabled
+if (profile) {
+  cliOut(`compile cache: status ${compileCache.status}` + ` ${compileCache.directory ?? compileCache.message ?? ''}`);
+}
+// hooks have to be registered before the command module - and so its import tree - is loaded
+const profileModulesArg = args.find(arg => arg === '--profile-modules' || arg.startsWith('--profile-modules='));
+if (profileModulesArg !== undefined) {
+  const {
+    registerModuleLoadAnalysis
+  } = await import("./moduleLoadAnalysis.js");
+  registerModuleLoadAnalysis(profileModulesArg.split('=')[1]);
+}
 switch (command) {
   case undefined:
   case '':
@@ -15,6 +37,7 @@ switch (command) {
   case 'dev':
     {
       cliOut(`Starting Tungsten dev mode`);
+      beginStartupProfiling(profile);
       // set env before loading the next file, to set the env before file imports
       process.env.__TUNGSTEN_ENV = 'development';
       await import("./commands/dev.js");
@@ -35,6 +58,7 @@ switch (command) {
       }
       // announce before the import: resolving the build's dependencies is a long silent wait
       if (!args.includes('--quiet')) cliOut(`Starting Tungsten production build`);
+      beginStartupProfiling(profile);
       process.env.__TUNGSTEN_ENV = 'production';
       process.env.__TUNGSTEN_DEBUG = 'true';
       await import("./commands/build.js");
@@ -61,6 +85,7 @@ switch (command) {
     }
   case 'compile':
     {
+      beginStartupProfiling(profile);
       process.env.__TUNGSTEN_ENV = 'development';
       await import("./commands/compile.js");
       break;
@@ -75,4 +100,13 @@ switch (command) {
       cliWarn(`tungsten: unknown command "${command}" (run "tungsten --help" for usage)`);
       process.exit(1);
     }
-}
+}
+// Node only writes the compile cache when the process exits, and dev/start are killed rather than
+// exited - signals skip exit handling - so they would populate the cache and then throw it away,
+// paying the cold cost on every run forever. The import above has returned by here, which for a
+// server command means it has finished starting, so this persists what that startup compiled.
+flushCompileCache();
+// A profiled dev run has measured everything it will measure once the server is up and the report
+// has printed. Exit rather than leave a server to be killed - after the flush above, which a kill
+// would have skipped. (dev.ts itself must not exit: its import returning is what reaches here.)
+if (profile && command === 'dev') process.exit(0);

package/dist/bin/moduleLoadAnalysis.d.ts

@@ -0,0 +1,2 @@
+export declare function registerModuleLoadAnalysis(pivotHint: string | undefined): void;
+//# sourceMappingURL=moduleLoadAnalysis.d.ts.map

package/dist/bin/moduleLoadAnalysis.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"moduleLoadAnalysis.d.ts","sourceRoot":"","sources":["../../src/bin/moduleLoadAnalysis.ts"],"names":[],"mappings":"AAmCA,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CA0B9E"}

package/dist/bin/moduleLoadAnalysis.js

@@ -0,0 +1,243 @@
+// --profile-modules: records the real import graph and per-module cost through node's module
+// hooks. Timing a subsystem on its own overstates it - it pays for everything it pulls in,
+// including what a sibling would have pulled in anyway - so this splits each subsystem's cost
+// into what only it needs and what it shares, and names who it shares with.
+import { registerHooks } from 'node:module';
+import { performance } from 'node:perf_hooks';
+import chalk from 'chalk-extensions';
+import { cliOut } from "../cliOutput.js";
+import { addEndOfRunReport } from "../timeProfiler.js";
+const records = new Map();
+// The user's app enters the graph through an iterationQueryString() import, so its first module
+// arrives as an explicitly tagged specifier - and recording stops there. Past that point the cost
+// is the app's own dependencies, which tungsten cannot do anything about. Matching the specifier
+// rather than the resolved url makes this independent of the order the registered hooks run in.
+const appModulePattern = /[?&]t-(?:instance|reload)\b/;
+let stoppedAtApp = false;
+function recordFor(url) {
+  const existing = records.get(url);
+  if (existing !== undefined) return existing;
+  const created = {
+    url,
+    importers: new Set(),
+    ms: 0
+  };
+  records.set(url, created);
+  return created;
+}
+export function registerModuleLoadAnalysis(pivotHint) {
+  registerHooks({
+    resolve(specifier, context, next) {
+      if (appModulePattern.test(specifier)) stoppedAtApp = true;
+      if (stoppedAtApp) return next(specifier, context);
+      const start = performance.now();
+      const result = next(specifier, context);
+      const entry = recordFor(result.url);
+      entry.ms += performance.now() - start;
+      if (context.parentURL !== undefined) entry.importers.add(context.parentURL);
+      return result;
+    },
+    load(url, context, next) {
+      if (stoppedAtApp) return next(url, context);
+      const start = performance.now();
+      const result = next(url, context);
+      recordFor(url).ms += performance.now() - start;
+      return result;
+    }
+  });
+  // Printed where each command reports its run total - by then everything is loaded, and dev
+  // has its server up rather than waiting to be killed. process.on('exit') cannot do this: a
+  // watcher ended with ctrl-c dies to the signal and never runs exit handlers. Kept as a
+  // fallback anyway, for a command that ends before it reports a total.
+  addEndOfRunReport(() => reportOnce(pivotHint));
+  process.on('exit', () => reportOnce(pivotHint));
+}
+let reported = false;
+function reportOnce(pivotHint) {
+  if (reported) return;
+  reported = true;
+  reportModuleLoadAnalysis(pivotHint);
+}
+function ms(value) {
+  return `${Math.round(value).toLocaleString()}ms`;
+}
+function shorten(url) {
+  const path = url.startsWith('file:///') ? url.slice('file:///'.length) : url;
+  const marker = path.lastIndexOf('node_modules/');
+  const name = marker !== -1 ? path.slice(marker + 'node_modules/'.length) : path.split('/').slice(-3).join('/');
+  return name.length > 46 ? `…${name.slice(-45)}` : name;
+}
+// the npm package a module belongs to - tungsten's own files and node builtins have none
+function packageOf(url) {
+  const marker = url.lastIndexOf('node_modules/');
+  if (marker === -1) return undefined;
+  const [scope, scoped] = url.slice(marker + 'node_modules/'.length).split('/');
+  if (scope === undefined) return undefined;
+  return scope.startsWith('@') ? `${scope}/${scoped ?? ''}` : scope;
+}
+// What dropping a package would actually save: everything reachable from it that nothing else
+// reaches - its own modules plus the transitive dependencies it alone pulls in. A package's total
+// size is the wrong number for that, because most of a small package can be dependencies it
+// shares with something you are keeping.
+function reportPackageCosts(loaded) {
+  const children = childGraph();
+  // Only packages entered from outside node_modules - tungsten's own code - are candidates.
+  // Counting nested packages too would leave every one of them at zero, since its modules are
+  // always reachable from whichever package depends on it as well as from itself.
+  const entries = new Map();
+  for (const entry of loaded) {
+    const name = packageOf(entry.url);
+    if (name === undefined) continue;
+    if (![...entry.importers].some(importer => packageOf(importer) === undefined)) continue;
+    entries.set(name, [...(entries.get(name) ?? []), entry.url]);
+  }
+  const owners = new Map();
+  for (const [name, urls] of entries) {
+    const reached = new Set();
+    for (const url of urls) for (const found of reachableFrom(url, children)) reached.add(found);
+    for (const url of reached) owners.set(url, [...(owners.get(url) ?? []), name]);
+  }
+  const totals = new Map();
+  for (const [url, names] of owners) {
+    const cost = costOf(url);
+    for (const name of names) {
+      const stat = totals.get(name) ?? {
+        exclusive: 0,
+        shared: 0,
+        modules: 0
+      };
+      if (names.length === 1) {
+        stat.exclusive += cost;
+        stat.modules += 1;
+      } else stat.shared += cost;
+      totals.set(name, stat);
+    }
+  }
+  cliOut('');
+  cliOut(chalk.mediumgrey`packages imported directly` + chalk.mediumgrey` - "only it" is what dropping the package would save, its own` + chalk.mediumgrey` modules plus the dependencies nothing else pulls in`);
+  cliOut('  ' + chalk.mediumgrey`${'package'.padEnd(30)}` + ' ' + chalk.mediumgrey`${'only it'.padStart(9)}` + ' ' + chalk.mediumgrey`${'shared'.padStart(9)}`);
+  const ranked = [...totals].sort((a, b) => b[1].exclusive - a[1].exclusive).slice(0, 15);
+  for (const [name, stat] of ranked) {
+    cliOut('  ' + chalk.yellow`${name.padEnd(30)}` + ' ' + chalk.orange`${ms(stat.exclusive).padStart(9)}` + ' ' + chalk.mediumgrey`${ms(stat.shared).padStart(9)}` + ' ' + chalk.mediumgrey`${stat.modules} modules only it needs`);
+  }
+}
+function childGraph() {
+  const children = new Map();
+  for (const entry of records.values()) {
+    for (const importer of entry.importers) {
+      const existing = children.get(importer);
+      if (existing === undefined) children.set(importer, new Set([entry.url]));else existing.add(entry.url);
+    }
+  }
+  return children;
+}
+function reachableFrom(root, children) {
+  const seen = new Set();
+  const stack = [root];
+  for (let url = stack.pop(); url !== undefined; url = stack.pop()) {
+    if (seen.has(url)) continue;
+    seen.add(url);
+    for (const child of children.get(url) ?? []) stack.push(child);
+  }
+  return seen;
+}
+// Framework.js is the interesting fork for dev/build; compile has no Framework, so fall back to
+// the command module. --profile-modules=<substring> overrides both.
+function choosePivot(pivotHint) {
+  const urls = [...records.keys()];
+  if (pivotHint !== undefined && pivotHint !== '') return urls.find(url => url.includes(pivotHint));
+  return urls.find(url => url.endsWith('/Framework.js')) ?? urls.find(url => url.includes('/commands/'));
+}
+function costOf(url) {
+  return records.get(url)?.ms ?? 0;
+}
+function reportModuleLoadAnalysis(pivotHint) {
+  // snapshot first: printing loads chalk's template module, which would otherwise land in the
+  // listing below but not in the count above it
+  const loaded = [...records.values()];
+  if (loaded.length === 0) return;
+  const recorded = loaded.reduce((total, entry) => total + entry.ms, 0);
+  cliOut('');
+  const scope = stoppedAtApp ? `, stopping where the user's app starts loading` : ``;
+  const summary = ` - ${loaded.length.toLocaleString()} modules, ${ms(recorded)} in resolve + read` + `${scope} (V8 compile and module evaluation are not covered)`;
+  cliOut(chalk.yellow`module load analysis` + chalk.mediumgrey`${summary}`);
+  // what bundling tungsten's own dist would collapse, as against what lives in node_modules
+  const own = loaded.filter(entry => !entry.url.startsWith('node:') && packageOf(entry.url) === undefined);
+  const ownMs = own.reduce((total, entry) => total + entry.ms, 0);
+  const dependencies = loaded.length - own.length;
+  cliOut(chalk.mediumgrey`  tungsten's own files: ${own.length.toLocaleString()} modules, ${ms(ownMs)}` + ` - dependencies: ${dependencies.toLocaleString()} modules, ${ms(recorded - ownMs)}`);
+  cliOut('');
+  cliOut(chalk.mediumgrey`heaviest modules`);
+  const heaviest = [...loaded].sort((a, b) => b.ms - a.ms).slice(0, 12);
+  for (const entry of heaviest) {
+    cliOut('  ' + chalk.yellow`${shorten(entry.url).padEnd(46)}` + ' ' + chalk.orange`${ms(entry.ms).padStart(9)}` + ' ' + chalk.mediumgrey`${`${entry.importers.size} importer${entry.importers.size === 1 ? '' : 's'}`}`);
+  }
+  reportPackageCosts(loaded);
+  const pivot = choosePivot(pivotHint);
+  if (pivot === undefined) return;
+  const children = childGraph();
+  const roots = [...(children.get(pivot) ?? [])].filter(url => !url.startsWith('node:'));
+  if (roots.length === 0) return;
+  // which roots can reach each module - one root means the cost is that root's alone
+  const owners = new Map();
+  for (const root of roots) {
+    for (const url of reachableFrom(root, children)) {
+      if (url === pivot) continue;
+      const existing = owners.get(url);
+      if (existing === undefined) owners.set(url, [root]);else existing.push(root);
+    }
+  }
+  const stats = new Map(roots.map(root => [root, {
+    exclusive: 0,
+    shared: 0
+  }]));
+  const groups = new Map();
+  let attributed = 0;
+  for (const [url, ownerList] of owners) {
+    const cost = costOf(url);
+    attributed += cost;
+    for (const root of ownerList) {
+      const stat = stats.get(root);
+      if (stat === undefined) continue;
+      if (ownerList.length === 1) stat.exclusive += cost;else stat.shared += cost;
+    }
+    if (ownerList.length === 1) continue;
+    const key = [...ownerList].sort().join('\n');
+    const group = groups.get(key);
+    if (group === undefined) groups.set(key, {
+      roots: [...ownerList].sort(),
+      ms: cost,
+      modules: 1
+    });else {
+      group.ms += cost;
+      group.modules += 1;
+    }
+  }
+  cliOut('');
+  cliOut(chalk.mediumgrey`direct imports of ${shorten(pivot)}` + chalk.mediumgrey` - "alone" is what that import would cost with no siblings loaded`);
+  cliOut('  ' + chalk.mediumgrey`${'import'.padEnd(46)}` + ' ' + chalk.mediumgrey`${'only it'.padStart(9)}` + ' ' + chalk.mediumgrey`${'shared'.padStart(9)}` + ' ' + chalk.mediumgrey`${'alone'.padStart(9)}`);
+  const ranked = roots.map(root => ({
+    root,
+    ...(stats.get(root) ?? {
+      exclusive: 0,
+      shared: 0
+    })
+  })).sort((a, b) => b.exclusive + b.shared - (a.exclusive + a.shared));
+  for (const {
+    root,
+    exclusive,
+    shared
+  } of ranked.slice(0, 15)) {
+    cliOut('  ' + chalk.yellow`${shorten(root).padEnd(46)}` + ' ' + chalk.orange`${ms(exclusive).padStart(9)}` + ' ' + chalk.mediumgrey`${ms(shared).padStart(9)}` + ' ' + chalk.mediumgrey`${ms(exclusive + shared).padStart(9)}`);
+  }
+  if (ranked.length > 15) cliOut('  ' + chalk.mediumgrey`+ ${ranked.length - 15} more`);
+  const outside = `${ms(recorded - attributed)} outside ${shorten(pivot)} (the cli entry and its own imports)`;
+  cliOut('  ' + chalk.mediumgrey`${outside}`);
+  const sharedGroups = [...groups.values()].sort((a, b) => b.ms - a.ms).slice(0, 8);
+  if (sharedGroups.length === 0) return;
+  cliOut('');
+  cliOut(chalk.mediumgrey`shared subtrees - counted once here, but in every sharer's "alone" above`);
+  for (const group of sharedGroups) {
+    cliOut('  ' + chalk.orange`${ms(group.ms).padStart(9)}` + ' ' + chalk.mediumgrey`${`${group.modules} modules`.padEnd(14)}` + ' ' + chalk.yellow`${group.roots.map(shorten).join(' + ')}`);
+  }
+}

package/dist/build/compileJavascript.d.ts

@@ -15,8 +15,10 @@ export type JavascriptCompilationCycle = {
 export type CompileJavascriptOptions = {
     quiet?: boolean;
     packagesCacheLocation?: string;
+    packages?: boolean;
     hmr?: boolean;
     onCycleComplete?: ((cycle: JavascriptCompilationCycle) => void | Promise<void>) | undefined;
+    tests?: boolean;
 };
 export declare function compileJavascript(projectDir: string, appConfig: FrameworkConfigComplete, codeLocation: string, sourceLocation: string, styleImportBase: string, buildLocation: string, buildCodeLocation: string, mode: Mode, watch?: boolean, options?: CompileJavascriptOptions): Promise<CompileJavascriptResult>;
 export {};

package/dist/build/compileJavascript.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compileJavascript.d.ts","sourceRoot":"","sources":["../../src/build/compileJavascript.ts"],"names":[],"mappings":"AAEA,OAAO,QAAQ,MAAM,UAAU,CAAA;AAO/B,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAA;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAA;AAetC,KAAK,uBAAuB,GAAG;IAC3B,OAAO,EAAE,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAA;IAClC,WAAW,EAAE,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAA;IACtC,aAAa,EAAE,aAAa,CAAA;IAC5B,cAAc,EAAE,OAAO,CAAA;CAC1B,CAAA;AAED,MAAM,MAAM,0BAA0B,GAAG;IACrC,OAAO,EAAE,OAAO,CAAA;IAChB,IAAI,CAAC,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,wBAAwB,GAAG;IACnC,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,GAAG,CAAC,EAAE,OAAO,CAAA;IACb,eAAe,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAA;CAC9F,CAAA;AAKD,wBAAsB,iBAAiB,CACnC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,uBAAuB,EACtD,YAAY,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EACrE,aAAa,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,UAAO,EAC1E,OAAO,GAAE,wBAA6B,GACvC,OAAO,CAAC,uBAAuB,CAAC,CA+YlC"}
+{"version":3,"file":"compileJavascript.d.ts","sourceRoot":"","sources":["../../src/build/compileJavascript.ts"],"names":[],"mappings":"AAGA,OAAO,QAAQ,MAAM,UAAU,CAAA;AAO/B,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAA;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAA;AAiBtC,KAAK,uBAAuB,GAAG;IAC3B,OAAO,EAAE,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAA;IAClC,WAAW,EAAE,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAA;IACtC,aAAa,EAAE,aAAa,CAAA;IAC5B,cAAc,EAAE,OAAO,CAAA;CAC1B,CAAA;AAED,MAAM,MAAM,0BAA0B,GAAG;IACrC,OAAO,EAAE,OAAO,CAAA;IAChB,IAAI,CAAC,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,wBAAwB,GAAG;IACnC,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAE9B,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,GAAG,CAAC,EAAE,OAAO,CAAA;IACb,eAAe,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAA;IAE3F,KAAK,CAAC,EAAE,OAAO,CAAA;CAClB,CAAA;AAoBD,wBAAsB,iBAAiB,CACnC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,uBAAuB,EACtD,YAAY,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EACrE,aAAa,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,UAAO,EAC1E,OAAO,GAAE,wBAA6B,GACvC,OAAO,CAAC,uBAAuB,CAAC,CA+blC"}

package/dist/build/compileJavascript.js

@@ -1,6 +1,7 @@
 import { join, dirname } from 'path';
+import { pathToFileURL } from 'url';
 import chokidar from 'chokidar';
-import { transformFileAsync as babeltransformFileAsync } from '@babel/core';
+import { transformFileSync as babelTransformFileSync } from '@babel/core';
 import fs, { promises as fsPromises } from 'fs';
 import slash from 'slash';
 import serializeJavascript from 'serialize-javascript';
@@ -8,13 +9,30 @@ import { globby } from 'globby';
 import { packagesCacheFolder } from "../settings.js";
 import { makeBabelUserDevConfig } from "./makeBabelUserDevConfig.js";
 import { makeBabelUserProdConfig } from "./makeBabelUserProdConfig.js";
-import { cliOut, cliLog, cliError } from "../cliOutput.js"; // eslint-disable-line @typescript-eslint/no-unused-vars
+import { cliOut, cliLog, cliError, cliWarn } from "../cliOutput.js"; // eslint-disable-line @typescript-eslint/no-unused-vars
+import { createTimeProfiler, isProfiling, tally } from "../timeProfiler.js";
+import { tallyBabelPlugins } from "./tallyBabelPlugins.js";
 import { TungstenError } from "../TungstenError.js";
 import { getFrameworkDir } from "../getFrameworkDir.js";
+// The graph writeDependencyGraph() left behind on the previous run, naming a compiled copy of every
+// package file in the packages cache. Missing or unreadable just means everything is compiled from
+// source, which is what happened on every run before this was read back.
+async function loadDependencyGraph(location) {
+  if (!fs.existsSync(location)) return {};
+  try {
+    const graphModule = await import(pathToFileURL(location).href);
+    return graphModule.graph ?? {};
+  } catch (error) {
+    cliWarn(`Could not read the package cache at ${location}, compiling packages from source`);
+    cliLog(error);
+    return {};
+  }
+}
 ///// needs work to work in prod mode
 ///// see babel config
 export async function compileJavascript(projectDir, appConfig, codeLocation, sourceLocation, styleImportBase, buildLocation, buildCodeLocation, mode, watch = true, options = {}) {
   if (options.quiet !== true) cliOut(`Compiling app code`);
+  const compilePackages = options.packages !== false;
   let dependencyGraphDir = '';
   let dependencyGraphLocation = '';
   let oldDependencyGraph = {};
@@ -36,15 +54,23 @@ export async function compileJavascript(projectDir, appConfig, codeLocation, sou
   if (mode === 'dev') {
     dependencyGraphDir = options.packagesCacheLocation ?? join(projectDir, packagesCacheFolder);
     dependencyGraphLocation = join(dependencyGraphDir, '.dependency-graph.js');
-    oldDependencyGraph = {};
+    oldDependencyGraph = await loadDependencyGraph(dependencyGraphLocation);
     dependencyGraph = {};
     importAliases = {};
     promises = [];
-    config = makeBabelUserDevConfig(projectDir, codeLocation, sourceLocation, styleImportBase, buildLocation, buildCodeLocation, dependencyGraphDir, skipPackageCompilationFiles, oldDependencyGraph, dependencyGraph, importAliases, promises, watch, options.hmr === true);
+    config = makeBabelUserDevConfig(projectDir, codeLocation, sourceLocation, styleImportBase, buildLocation, buildCodeLocation, dependencyGraphDir, skipPackageCompilationFiles, oldDependencyGraph, dependencyGraph, importAliases, promises, watch, options.hmr === true, compilePackages);
   } else {
     config = makeBabelUserProdConfig(projectDir, codeLocation, sourceLocation, styleImportBase, watch);
   }
   config.root = getFrameworkDir();
+  // per-plugin attribution of the transform's time, only when someone is looking: the wrappers
+  // run per AST node
+  if (isProfiling()) {
+    config.plugins = tallyBabelPlugins(config.plugins);
+    if (mode === 'dev' && compilePackages) {
+      cliWarn(`profiling serves linked packages from the packages cache -` + ` delete ${dependencyGraphDir} after rebuilding one, or this run compiles stale code`);
+    }
+  }
   // compile with the API
   const codeLocationLength = codeLocation.length + 1;
   const transformPromises = [];
@@ -67,7 +93,13 @@ export async function compileJavascript(projectDir, appConfig, codeLocation, sou
         failedFiles.delete(file);
         return;
       }
-      const result = await babeltransformFileAsync(file, config);
+      // Sync, so this can be tallied exactly - nothing else runs during it. Babel's work is
+      // cpu-bound on one thread either way, so the async API's 291 concurrent calls bought
+      // no parallelism, only scheduling. The plugins defer their async work into
+      // opts.promises rather than awaiting in a visitor, which is what makes sync viable.
+      const babelStart = performance.now();
+      const result = babelTransformFileSync(file, config);
+      tally('babel transform', performance.now() - babelStart);
       if (result === null) throw new TungstenError(`Err: babel transform on file ${file} returned null`);
       const {
         code
@@ -174,6 +206,8 @@ export async function compileJavascript(projectDir, appConfig, codeLocation, sou
   }
   // get a list of files
   const watchGlob = slash(`${codeLocation}/**/*.js{,x}`);
+  // typed rather than inferred from the pushes: it is captured by a closure below, which stops
+  // typescript widening it from the pushes
   const ignored = [];
   ignored.push(slash(`${buildLocation}/**`));
   if (appConfig.ignoreDirs !== undefined && appConfig.ignoreDirs !== '') {
@@ -189,6 +223,17 @@ export async function compileJavascript(projectDir, appConfig, codeLocation, sou
       ignored.push(appConfig.ignorePatterns);
     }
   }
+  // test files are a category, not noise: excluded here like ignoreDirs/ignorePatterns, but a
+  // test compile (`tungsten compile --tests`) wants them, where it would never want the ignored
+  if (options.tests !== true) {
+    if (appConfig.testDirs !== undefined && appConfig.testDirs !== '') {
+      const testDirs = typeof appConfig.testDirs === 'string' ? [appConfig.testDirs] : appConfig.testDirs;
+      ignored.push(...testDirs.map(folder => slash(`${codeLocation}/${folder}/**`)));
+    }
+    if (appConfig.testPatterns !== undefined && appConfig.testPatterns !== '') {
+      ignored.push(...(Array.isArray(appConfig.testPatterns) ? appConfig.testPatterns : [appConfig.testPatterns]));
+    }
+  }
   if (appConfig.exclude !== undefined) ignored.push(slash(`${codeLocation}/${appConfig.exclude}`));
   //console.log('appConfig', appConfig)
   //console.log('ignored', ignored)
@@ -197,11 +242,18 @@ export async function compileJavascript(projectDir, appConfig, codeLocation, sou
   //cliWarn(JSON.stringify(ignored))
   /// need to do an initial build before watching
   // not sure what glob matcher chokidar uses
-  const paths = await globby([watchGlob, ...ignored.map(x => `!${x}`)]);
-  paths.forEach(file => {
-    transformPromises.push(transformFile(file));
+  // phases record into the profiler module's shared timeline, so this instance need not be the
+  // one the command created - it only has to be enabled when --profile is
+  const profiler = createTimeProfiler(isProfiling());
+  const paths = await profiler.timeProfileAsync('find source files', () => globby([watchGlob, ...ignored.map(x => `!${x}`)]));
+  // the transforms are started inside the phase, not just awaited there: a resource is charged to
+  // the phase that created it, so starting them outside credits all the work to the parent
+  const transformResults = await profiler.timeProfileAsync(`transform ${paths.length} files`, () => {
+    paths.forEach(file => {
+      transformPromises.push(transformFile(file));
+    });
+    return Promise.allSettled(transformPromises);
   });
-  const transformResults = await Promise.allSettled(transformPromises);
   const transformFailures = transformResults.filter(result => result.status === 'rejected').map(result => result.reason);
   const settleDeferredWork = async () => {
     // splice out (rather than slice+advance-a-pointer) so settled entries are freed for GC -
@@ -211,10 +263,14 @@ export async function compileJavascript(projectDir, appConfig, codeLocation, sou
     const deferredResults = await Promise.allSettled(currentPromises);
     return deferredResults.filter(result => result.status === 'rejected').map(result => result.reason);
   };
-  const initialFailures = [...transformFailures, ...(await settleDeferredWork())];
+  // the package walk the transforms queued - compiling, or now copying from the packages cache,
+  // every file of every imported package
+  const deferredFailures = await profiler.timeProfileAsync('settle package work', settleDeferredWork);
+  const initialFailures = [...transformFailures, ...deferredFailures];
   const initialSuccess = initialFailures.length === 0 && failedFiles.size === 0;
   function writeDependencyGraph() {
-    if (mode !== 'dev') return;
+    // without the package walk the graph is empty - don't create the cache dir to hold it
+    if (mode !== 'dev' || !compilePackages) return;
     if (!fs.existsSync(dependencyGraphDir)) {
       fs.mkdirSync(dependencyGraphDir, {
         recursive: true
@@ -318,7 +374,9 @@ export async function compileJavascript(projectDir, appConfig, codeLocation, sou
       }
     });
   }
-  return new Promise(resolve => {
+  // the watchers are created above but only scan the tree now - on a large app that scan is
+  // seconds, and it runs after every file has already been compiled
+  return profiler.timeProfileAsync('watchers ready', () => new Promise(resolve => {
     const awaiting = new Map([[watcher, true]]);
     if (copyWatcher) awaiting.set(copyWatcher, true);
     watcher.on('ready', async () => {
@@ -348,5 +406,5 @@ export async function compileJavascript(projectDir, appConfig, codeLocation, sou
         }
       });
     }
-  });
+  }));
 }