peachsim 0.1.577 → 0.1.579

package/src/swift-compile.ts

@@ -10,19 +10,41 @@ import {
   statSync,
   writeFileSync,
 } from 'node:fs'
+import os from 'node:os'
 import path from 'node:path'
 import { promisify } from 'node:util'
-import { swiftManifestTargets, type SwiftPackageTarget } from './swift-package'
+import { swiftIcuAsset, type SwiftIcuAsset } from './swift-icu'
+import {
+  swiftManifestTargets,
+  swiftPackagePartition,
+  type SwiftPackageTarget,
+} from './swift-package'
 import type { SwiftImageUpload } from './swift-submit'
 
+export { swiftIcuAsset, type SwiftIcuAsset }
+
 const run = promisify(execFile)
 const WASM_TRIPLE = 'wasm32-unknown-wasip1'
 
+// the link runs wasm-opt, which threads across every core (970% peak on 16
+// cores for the counter fixture) and pins a shared machine for each compile.
+// binaryen's own cap holds it to 4 threads (400%, 26 -> 32 s); swiftpm's jobs
+// take half the cores, as the fixture build does.
+const cores = os.availableParallelism()
+const BUILD_ENV = { ...process.env, BINARYEN_CORES: String(Math.min(4, cores)) }
+const BUILD_JOBS = String(Math.max(1, Math.floor(cores / 2)))
+
 export interface SwiftCompileInput {
   files: Record<string, string>
   resources?: Record<string, string>
   images?: Record<string, SwiftImageUpload>
   entry: string
+  /**
+   * the sources are a One swift package. One's pod compiles every file with
+   * `-import-module One`, so `RNXModule`, `RNXPackage` and `JSON` resolve
+   * without an import; the compile links Peach's `One` module the same way.
+   */
+  oneModule?: boolean
 }
 
 export interface SwiftCompileOptions {
@@ -36,6 +58,8 @@ export interface SwiftCompileOptions {
 export interface SwiftCompileArtifact {
   hash: string
   bytes: Buffer
+  /** the icu common data the artifact reserves and its instantiator fills. */
+  icu: SwiftIcuAsset
 }
 
 export interface SwiftProjectLayout {
@@ -44,19 +68,62 @@ export interface SwiftProjectLayout {
   multi: boolean
 }
 
+// toolchain.json is an input for its wasm sdk checksum: the sdk's icu blob,
+// which the host fills into every artifact, is part of what the address names.
 export function swiftModuleInputFiles(root: string): string[] {
-  const files: string[] = ['Package.swift', 'scripts/contrast-app-Package.swift']
+  const files: string[] = [
+    'Package.swift',
+    'scripts/contrast-app-Package.swift',
+    'toolchain.json',
+  ]
   const walk = (dir: string) => {
     for (const entry of readdirSync(path.join(root, dir), { withFileTypes: true })) {
       const relative = `${dir}/${entry.name}`
       if (entry.isDirectory()) walk(relative)
-      else if (entry.name.endsWith('.swift')) files.push(relative)
+      else if (/\.(swift|c|h)$/.test(entry.name)) files.push(relative)
     }
   }
   walk('Sources')
   return files.sort()
 }
 
+// the host halves of the module's macros: the rnx plugin (#Preview, @Entry,
+// @Model) and #Predicate's FoundationMacros stand-in, which one script builds
+// side by side. manifest swiftSettings cannot load a plugin library (RAN: the
+// flag reaches the driver but the plugin never loads), so every build loads
+// both with -Xswiftc, as scripts/build.sh does for the fixtures. their
+// sources are under Sources/, which the artifact hash already covers.
+async function ensureMacroPlugins(
+  moduleRoot: string,
+  toolchainBin: string,
+): Promise<string[]> {
+  const dir = path.join(moduleRoot, '.build/host/RnxPreviewMacros')
+  // linux names a plugin's module from lib<Module>.so
+  const ext = process.platform === 'darwin' ? 'dylib' : 'so'
+  const plugins = [
+    {
+      library: path.join(dir, `libRnxPreviewMacros.${ext}`),
+      source: path.join(moduleRoot, 'Sources/RnxPreviewMacros/PreviewMacro.swift'),
+    },
+    {
+      library: path.join(dir, `libFoundationMacros.${ext}`),
+      source: path.join(moduleRoot, 'Sources/FoundationMacros/PredicateMacro.swift'),
+    },
+  ]
+  const stale = plugins.some(
+    ({ library, source }) =>
+      !existsSync(library) || statSync(source).mtimeMs > statSync(library).mtimeMs,
+  )
+  if (stale) {
+    await run('sh', [
+      path.join(moduleRoot, 'scripts/build-preview-plugin.sh'),
+      toolchainBin,
+      plugins[0].library,
+    ])
+  }
+  return plugins.map(({ library }) => library)
+}
+
 const linkedModuleDigests = new Map<string, { stamp: string; digest: Buffer }>()
 
 export function clearLinkedModuleDigests(root?: string): void {
@@ -93,7 +160,32 @@ function linkedModuleDigest(root: string): Buffer {
 
 export function compileTargets(input: SwiftCompileInput): SwiftPackageTarget[] {
   const manifest = input.files['Package.swift']
-  return manifest === undefined ? [] : swiftManifestTargets(manifest)
+  return manifest === undefined ? [] : swiftPackagePartition(manifest)
+}
+
+/**
+ * the compile input with a package's generated native dispatch glue in its
+ * one target, linked against the One module the glue conforms to. vxrn's
+ * `renderSwiftSourceGlue` renders the glue; a package with native modules
+ * across several targets has no single place for it.
+ */
+export function withNativeSourceGlue(
+  input: SwiftCompileInput,
+  glue: string,
+): SwiftCompileInput {
+  const targets = compileTargets(input)
+  if (targets.length > 1) {
+    throw new Error('swift native modules need one SwiftPM target')
+  }
+  const targetPath = targets[0]?.path === '.' ? '' : (targets[0]?.path ?? '')
+  return {
+    ...input,
+    files: {
+      ...input.files,
+      [path.posix.join(targetPath, 'OneNativeSource.generated.swift')]: glue,
+    },
+    oneModule: true,
+  }
 }
 
 function swiftSiblingDependencies(
@@ -107,7 +199,11 @@ function swiftSiblingDependencies(
     const deps = new Set<string>()
     for (const [file, contents] of Object.entries(files)) {
       if (file === 'Package.swift' || !file.startsWith(prefix)) continue
-      for (const match of contents.matchAll(/^\s*import\s+([A-Za-z_][A-Za-z0-9_]*)/gm)) {
+      // an import may carry attributes (`@_exported`, `@preconcurrency`) and a
+      // declaration kind (`import struct Nuke.ImageRequest`).
+      for (const match of contents.matchAll(
+        /^\s*(?:@\w+(?:\([^)]*\))?\s+)*import\s+(?:(?:typealias|struct|class|enum|protocol|let|var|func)\s+)?([A-Za-z_][A-Za-z0-9_]*)/gm,
+      )) {
         if (names.has(match[1]) && match[1] !== target.name) deps.add(match[1])
       }
     }
@@ -116,8 +212,11 @@ function swiftSiblingDependencies(
   return siblings
 }
 
+const ONE_PRODUCT = '.product(name: "One", package: "sootsim-swift")'
+const ONE_IMPLICIT_FLAGS = ', "-Xfrontend", "-import-module", "-Xfrontend", "One"'
+
 /** bump with any change to the emitted multi-target manifest text. */
-export const MULTI_MANIFEST_VERSION = 1
+export const MULTI_MANIFEST_VERSION = 5
 
 export function swiftArtifactHash(
   input: SwiftCompileInput,
@@ -126,6 +225,7 @@ export function swiftArtifactHash(
 ): string {
   const hash = createHash('sha256')
   hash.update(`swift-compile/v1\n${release}\n`)
+  if (input.oneModule) hash.update('one-module\n')
   hash.update(linkedModuleDigest(moduleRoot))
   for (const file of Object.keys(input.files).sort()) {
     hash.update(createHash('sha256').update(file).digest())
@@ -165,15 +265,21 @@ export function emitSwiftPackageManifest(args: {
   targets: SwiftPackageTarget[]
   siblings: Map<string, string[]>
   moduleRoot: string
+  oneModule?: boolean
 }): string {
-  const { targets, siblings, moduleRoot } = args
+  const { targets, siblings, moduleRoot, oneModule } = args
   const executable = targets.find((target) => target.executable)
   if (!executable) throw new Error('swift-compile: no executable target to emit')
-  const swiftSettings =
-    'swiftSettings: [.swiftLanguageMode(.v5), .unsafeFlags(["-Osize"])]'
+  // each target compiles in the language mode and default isolation its
+  // author declared, the way xcode builds the same package.
   const stanza = (target: SwiftPackageTarget): string => {
+    const isolation = target.mainActorIsolation
+      ? ' .defaultIsolation(MainActor.self),'
+      : ''
+    const swiftSettings = `swiftSettings: [.swiftLanguageMode(.v${target.languageMode}),${isolation} .unsafeFlags(["-Osize"${oneModule ? ONE_IMPLICIT_FLAGS : ''}])]`
     const deps = [
       '.product(name: "SwiftUI", package: "sootsim-swift")',
+      ...(oneModule ? [ONE_PRODUCT] : []),
       ...(siblings.get(target.name) ?? []).map((sibling) => `"${sibling}"`),
     ].join(', ')
     const kind = target.executable ? '.executableTarget' : '.target'
@@ -196,8 +302,10 @@ export function emitSwiftPackageManifest(args: {
       `    ),`,
     ].join('\n')
   }
+  // 6.2 is the first tools version with `.defaultIsolation`, which a target
+  // carries over from the project manifest.
   return [
-    '// swift-tools-version: 6.0',
+    '// swift-tools-version: 6.2',
     'import PackageDescription',
     '',
     '// generated by swift-compile from the project package partition.',
@@ -213,46 +321,74 @@ export function emitSwiftPackageManifest(args: {
   ].join('\n')
 }
 
+// the build tree mirrors the input: a file is written only when its text
+// changed and a file the input dropped is removed, so an unchanged source
+// keeps its mtime and swiftpm rebuilds only what moved. `.build` stays.
+function syncSwiftTree(base: string, files: Map<string, string>): void {
+  mkdirSync(base, { recursive: true })
+  const walk = (dir: string): void => {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const full = path.join(dir, entry.name)
+      if (dir === base && entry.name === '.build') continue
+      if (entry.isDirectory()) walk(full)
+      else if (!files.has(full)) rmSync(full, { force: true })
+    }
+  }
+  walk(base)
+  for (const [full, contents] of files) {
+    if (existsSync(full) && readFileSync(full, 'utf8') === contents) continue
+    mkdirSync(path.dirname(full), { recursive: true })
+    writeFileSync(full, contents, 'utf8')
+  }
+}
+
 export function writeSwiftProject(
   layout: SwiftProjectLayout,
   input: SwiftCompileInput,
   moduleRoot: string,
 ): void {
   const targets = compileTargets(input)
+  const files = new Map<string, string>()
   if (targets.length === 0) {
-    rmSync(layout.sources, { recursive: true, force: true })
     for (const [file, contents] of Object.entries(input.files)) {
-      const target = path.join(layout.sources, file)
-      mkdirSync(path.dirname(target), { recursive: true })
-      writeFileSync(target, contents, 'utf8')
+      if (file === 'Package.swift') continue
+      files.set(path.join(layout.sources, file), contents)
     }
-    const manifest = readFileSync(
+    let manifest = readFileSync(
       path.join(moduleRoot, 'scripts/contrast-app-Package.swift'),
       'utf8',
     ).replaceAll('__SOOTSIM_SWIFT_PACKAGE__', moduleRoot)
-    writeFileSync(path.join(layout.root, 'Package.swift'), manifest, 'utf8')
-    return
-  }
-  mkdirSync(layout.root, { recursive: true })
-  for (const child of readdirSync(layout.root)) {
-    if (child === '.build') continue
-    rmSync(path.join(layout.root, child), { recursive: true, force: true })
-  }
-  for (const [file, contents] of Object.entries(input.files)) {
-    if (file === 'Package.swift') continue
-    const target = path.join(layout.root, file)
-    mkdirSync(path.dirname(target), { recursive: true })
-    writeFileSync(target, contents, 'utf8')
+    if (input.oneModule) {
+      // anchored on lines the template already has, so the template stays a
+      // valid manifest for a compiler build that predates the One module
+      const product = '.product(name: "CoreLocation", package: "sootsim-swift"),'
+      const flags = '"-Osize"'
+      if (!manifest.includes(product) || !manifest.includes(flags)) {
+        throw new Error(
+          'scripts/contrast-app-Package.swift lost the lines the One module links at',
+        )
+      }
+      manifest = manifest
+        .replace(product, `${product}\n        ${ONE_PRODUCT},`)
+        .replace(flags, `${flags}${ONE_IMPLICIT_FLAGS}`)
+    }
+    files.set(path.join(layout.root, 'Package.swift'), manifest)
+  } else {
+    for (const [file, contents] of Object.entries(input.files)) {
+      if (file === 'Package.swift') continue
+      files.set(path.join(layout.root, file), contents)
+    }
+    files.set(
+      path.join(layout.root, 'Package.swift'),
+      emitSwiftPackageManifest({
+        targets,
+        siblings: swiftSiblingDependencies(input.files, targets),
+        moduleRoot,
+        oneModule: input.oneModule,
+      }),
+    )
   }
-  writeFileSync(
-    path.join(layout.root, 'Package.swift'),
-    emitSwiftPackageManifest({
-      targets,
-      siblings: swiftSiblingDependencies(input.files, targets),
-      moduleRoot,
-    }),
-    'utf8',
-  )
+  syncSwiftTree(layout.root, files)
 }
 
 function streamText(value: unknown): string {
@@ -274,17 +410,27 @@ function outputOf(error: unknown): string {
 
 function builtArtifact(layout: SwiftProjectLayout): string {
   const binDir = path.join(layout.root, '.build', WASM_TRIPLE, 'release')
-  const artifacts = existsSync(binDir)
-    ? readdirSync(binDir)
-        .filter((name) => name.endsWith('.wasm'))
-        .sort()
-    : []
-  if (artifacts.length !== 1) {
+  // the build root is shared across projects so the SwiftUI module stays
+  // warm, and every project's executable lands beside the others. the
+  // written manifest names this build's executable, so select it by name
+  // instead of requiring the directory to hold exactly one artifact.
+  const manifest = readFileSync(path.join(layout.root, 'Package.swift'), 'utf8')
+  const executable = swiftManifestTargets(manifest).find((target) => target.executable)
+  if (executable === undefined) {
+    throw new Error('swift-compile: the written manifest names no executable target')
+  }
+  const file = path.join(binDir, `${executable.name}.wasm`)
+  if (!existsSync(file)) {
+    const found = existsSync(binDir)
+      ? readdirSync(binDir)
+          .filter((name) => name.endsWith('.wasm'))
+          .sort()
+      : []
     throw new Error(
-      `swift-compile: the build produced ${artifacts.length} wasm artifacts; exactly one executable is the app`,
+      `swift-compile: the build produced no ${executable.name}.wasm (found: ${found.join(', ') || 'none'})`,
     )
   }
-  return path.join(binDir, artifacts[0])
+  return file
 }
 
 function projectDiagnostic(output: string, layout: SwiftProjectLayout): string {
@@ -335,11 +481,17 @@ export function compileSwiftProjectUnit(
         `swift-compile: no swift.org toolchain at ${options.toolchainBin}; install ${options.release} and its matching wasm sdk`,
       )
     }
+    const macroPlugins = await ensureMacroPlugins(moduleRoot, options.toolchainBin)
+    // memoized per toolchain, so only a process's first build waits on it,
+    // and that one reads and gzips it while swift builds
+    const icu = swiftIcuAsset(options.toolchainBin, options.release)
     try {
       await run(
         swift,
         [
           'build',
+          '-j',
+          BUILD_JOBS,
           '--package-path',
           root,
           '--swift-sdk',
@@ -348,12 +500,22 @@ export function compileSwiftProjectUnit(
           'release',
           '-Xswiftc',
           '-Osize',
+          ...macroPlugins.flatMap((plugin) => [
+            '-Xswiftc',
+            '-load-plugin-library',
+            '-Xswiftc',
+            plugin,
+          ]),
         ],
-        { maxBuffer: 64 * 1024 * 1024, timeout: options.timeoutMs },
+        { env: BUILD_ENV, maxBuffer: 64 * 1024 * 1024, timeout: options.timeoutMs },
       )
     } catch (error) {
       throw new Error(projectDiagnostic(outputOf(error), layout), { cause: error })
     }
-    return { hash, bytes: readFileSync(builtArtifact(layout)) }
+    return {
+      hash,
+      bytes: readFileSync(builtArtifact(layout)),
+      icu: await icu,
+    }
   })
 }

package/src/swift-icu.ts

@@ -0,0 +1,140 @@
+import { execFile } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { readFileSync } from 'node:fs'
+import path from 'node:path'
+import { promisify } from 'node:util'
+import { gzip } from 'node:zlib'
+
+/**
+ * the icu common data a swift artifact reserves and its host fills
+ * (packages/sootsim-swift/Sources/PeachICU/icu.c). it is read straight from
+ * the wasm sdk the artifact links against, the archive member the link now
+ * leaves out, so it is exactly the bytes the linker used to place: one blob
+ * per toolchain release.
+ *
+ * node builtins only: the fixture harnesses import this file straight from
+ * the checkout.
+ */
+export interface SwiftIcuAsset {
+  /** sha256 of `gzip`, the address a host serves it at. */
+  sha256: string
+  /**
+   * the blob gzipped, which is how every host stores and serves it: 34 MB
+   * raw is 13 MB gzipped, and cloudflare compresses no binary type on its
+   * own. the tenant inflates it with DecompressionStream.
+   */
+  gzip: Buffer
+}
+
+const run = promisify(execFile)
+const deflate = promisify(gzip)
+const ARCHIVE = 'wasi/lib_FoundationICU.a'
+const MEMBER = 'icu_packaged_data.cpp.obj'
+const raw = new Map<string, Promise<Buffer>>()
+const assets = new Map<string, Promise<SwiftIcuAsset>>()
+
+function memo<T>(
+  cache: Map<string, Promise<T>>,
+  key: string,
+  load: () => Promise<T>,
+): Promise<T> {
+  let pending = cache.get(key)
+  if (!pending) {
+    pending = load()
+    pending.catch(() => cache.delete(key))
+    cache.set(key, pending)
+  }
+  return pending
+}
+
+/** the raw blob, for a harness that fills an artifact in node. */
+export function swiftIcuData(toolchainBin: string, release: string): Promise<Buffer> {
+  return memo(raw, `${toolchainBin}\0${release}`, () => load(toolchainBin, release))
+}
+
+/** the served form: gzipped and content-addressed. */
+export function swiftIcuAsset(
+  toolchainBin: string,
+  release: string,
+): Promise<SwiftIcuAsset> {
+  return memo(assets, `${toolchainBin}\0${release}`, async () => {
+    const zipped = await deflate(await swiftIcuData(toolchainBin, release))
+    return { sha256: createHash('sha256').update(zipped).digest('hex'), gzip: zipped }
+  })
+}
+
+async function load(toolchainBin: string, release: string): Promise<Buffer> {
+  const { stdout } = await run(path.join(toolchainBin, 'swift'), [
+    'sdk',
+    'configure',
+    '--show-configuration',
+    `${release}_wasm`,
+    'wasm32-unknown-wasip1',
+  ])
+  const resources = /^swiftStaticResourcesPath: (.+)$/m.exec(stdout)?.[1]?.trim()
+  if (!resources) throw new Error(`swift-icu: ${release}_wasm names no static resources`)
+  // a copy, so the 42 MB archive it was sliced from can be collected
+  return Buffer.from(
+    icuSegment(archiveMember(readFileSync(path.join(resources, ARCHIVE)), MEMBER)),
+  )
+}
+
+// a gnu ar archive: 60-byte headers, names over 15 characters live in the
+// `//` table and the header says `/<offset>`, members pad to even offsets.
+function archiveMember(archive: Buffer, name: string): Buffer {
+  if (archive.toString('latin1', 0, 8) !== '!<arch>\n')
+    throw new Error('swift-icu: not an ar archive')
+  let names = ''
+  for (let at = 8; at + 60 <= archive.length; ) {
+    const raw = archive.toString('latin1', at, at + 16).trimEnd()
+    const size = Number(archive.toString('latin1', at + 48, at + 58).trim())
+    const body = archive.subarray(at + 60, at + 60 + size)
+    if (raw === '//') names = body.toString('latin1')
+    const member = /^\/\d+$/.test(raw)
+      ? names.slice(Number(raw.slice(1)), names.indexOf('/\n', Number(raw.slice(1))))
+      : raw.replace(/\/$/, '')
+    if (member === name) return body
+    at += 60 + size + (size % 2)
+  }
+  throw new Error(`swift-icu: ${name} is not in the sdk's icu archive`)
+}
+
+// the relocatable object's one data segment whose bytes open with icu's
+// data-header magic (0xda27): the packaged data carries no relocations, so
+// the segment is the blob.
+function icuSegment(object: Buffer): Buffer {
+  let at = 8
+  const leb = (): number => {
+    let result = 0
+    let shift = 0
+    let byte: number
+    do {
+      byte = object[at++]
+      result += (byte & 0x7f) * 2 ** shift
+      shift += 7
+    } while (byte & 0x80)
+    return result
+  }
+  const found: Buffer[] = []
+  while (at < object.length) {
+    const id = object[at++]
+    const size = leb()
+    const end = at + size
+    if (id === 11) {
+      for (let count = leb(); count > 0; count--) {
+        if (leb() !== 0) throw new Error('swift-icu: unexpected data segment kind')
+        at++ // i32.const
+        leb()
+        at++ // end
+        const length = leb()
+        const bytes = object.subarray(at, at + length)
+        if (bytes[2] === 0xda && bytes[3] === 0x27) found.push(bytes)
+        at += length
+      }
+    }
+    at = end
+  }
+  if (found.length !== 1)
+    throw new Error(`swift-icu: expected one icu data segment, found ${found.length}`)
+  return found[0]
+}

package/src/swift-package.ts

@@ -6,6 +6,10 @@ export interface SwiftPackageTarget {
   path: string
   /** true for `.executableTarget`: the one that links the wasm artifact. */
   executable: boolean
+  /** the target's declared `.swiftLanguageMode(.vN)`; 5 when it declares none. */
+  languageMode: number
+  /** true when the target declares `.defaultIsolation(MainActor.self)`. */
+  mainActorIsolation: boolean
 }
 
 /** whether a project-relative key is a SwiftPM manifest, not a source. */
@@ -31,7 +35,8 @@ export function swiftManifestTargets(manifest: string): SwiftPackageTarget[] {
   while ((match = pattern.exec(code)) !== null) {
     const executable = match[1] === 'executableTarget'
     let depth = 1
-    let i = pattern.lastIndex
+    const start = pattern.lastIndex
+    let i = start
     let inString = false
     let escaped = false
     let flat = ''
@@ -63,7 +68,26 @@ export function swiftManifestTargets(manifest: string): SwiftPackageTarget[] {
       rawPath == null || rawPath === ''
         ? `Sources/${name}`
         : rawPath.replace(/^\.\//, '').replace(/\/+$/, '')
-    targets.push({ name, path: targetPath, executable })
+    // settings sit inside nested calls, which `flat` drops, so they are read
+    // from the call's whole text; a dependency list cannot spell either one.
+    const call = code.slice(start, i)
+    const languageMode = Number(
+      /\.swiftLanguageMode\(\s*\.v(\d+)\s*\)/.exec(call)?.[1] ?? '5',
+    )
+    const mainActorIsolation = /\.defaultIsolation\(\s*MainActor\.self\s*\)/.test(call)
+    targets.push({ name, path: targetPath, executable, languageMode, mainActorIsolation })
   }
   return targets
 }
+
+/**
+ * the target partition a root manifest declares for the compile. a single
+ * executable target at the package root (`path: "."`) holds the whole project,
+ * which is the compiler's single-target shape, so it declares no partition.
+ */
+export function swiftPackagePartition(manifest: string): SwiftPackageTarget[] {
+  const targets = swiftManifestTargets(manifest)
+  return targets.length === 1 && targets[0].executable && targets[0].path === '.'
+    ? []
+    : targets
+}