wyvrnpm 2.0.2 → 2.1.0

package/src/build/index.js

@@ -0,0 +1,244 @@
+'use strict';
+
+const fs   = require('fs');
+const path = require('path');
+const AdmZip = require('adm-zip');
+
+const { cloneAtSha }           = require('./clone');
+const { normalizeRecipe, hasBuildRecipe } = require('./recipe');
+const { configureBuildInstall } = require('./cmake');
+const { getBuildPaths, hasValidArtefact } = require('./cache');
+
+const { normalizeDependencies } = require('../manifest');
+const { resolveDependencies }   = require('../resolve');
+const { hashProfile, mergeProfile, sha256Of } = require('../profile');
+const { generateToolchain }     = require('../toolchain');
+const log                       = require('../logger');
+
+/**
+ * Recursively walk a directory and add every file into an AdmZip at
+ * paths relative to `base`. Matches the structure emitted by `publish` —
+ * zip contents are the DIRECT SUBTREE of `base`, not `base` itself.
+ */
+function addFolder(zip, dir, base) {
+  for (const entry of fs.readdirSync(dir)) {
+    const full = path.join(dir, entry);
+    const rel  = path.relative(base, full).replace(/\\/g, '/');
+    const stat = fs.statSync(full);
+    if (stat.isDirectory()) {
+      addFolder(zip, full, base);
+    } else {
+      const zipDir = path.dirname(rel);
+      zip.addLocalFile(full, zipDir === '.' ? '' : zipDir);
+    }
+  }
+}
+
+/**
+ * Build a package from source and produce an artefact zip indistinguishable
+ * from a pre-built download.
+ *
+ * ───────────────────────────────────────────────────────────────────────────
+ * Flow (PLAN-BUILD-MISSING phase 3):
+ *   1. Check the build cache for a ready artefact → copy and return.
+ *   2. Shallow-clone `{gitRepo}` at `{gitSha}` into the cache.
+ *   3. Read the cloned `wyvrn.json`; require a `build` section.
+ *   4. Recursively resolve + download the package's OWN dependencies into a
+ *      scratch `wyvrn_internal/` co-located with the source. Source-build
+ *      propagates via `buildMode: "missing"`, so transitive source-only deps
+ *      Just Work.
+ *   5. Generate the CMake toolchain via `generateToolchain()` — the shared
+ *      entry point documented in PLAN-CMAKE-TOOLCHAIN Phase F. Never hand-
+ *      roll profile→flag translation here.
+ *   6. Configure + build + install via `configureBuildInstall()`.
+ *   7. Zip the install dir → `wyvrn.zip`, compute `contentSha256`.
+ *   8. Copy the zip to `destZipPath` so `downloadDependencies` can extract
+ *      it like any other v2 artefact.
+ * ───────────────────────────────────────────────────────────────────────────
+ *
+ * @param {object} args
+ * @param {string} args.name
+ * @param {string} args.version
+ * @param {import('../profile').WyvrnProfile} args.profile
+ * @param {string} args.profileName
+ * @param {string} args.profileHash
+ * @param {string} args.gitRepo
+ * @param {string} args.gitSha
+ * @param {string} args.destZipPath    Where the caller expects the artefact zip.
+ * @param {string[]} args.packageSources  Used to resolve this package's own deps.
+ * @param {string} args.platform       v1 fallback platform for transitive deps.
+ * @param {number} args.timeoutMs
+ * @param {string} [args.buildMode]    Propagated into transitive downloads. Default 'missing'.
+ * @param {string[]} [args.configs]    Override recipe.configs (which defaults to
+ *                                      all four CMake configurations).
+ * @param {object} [args.authOptions]
+ * @returns {Promise<{
+ *   found:          true,
+ *   contentSha256:  string,
+ *   profileHash:    string,
+ *   gitSha:         string,
+ *   gitRepo:        string,
+ *   source:         string,       // build-cache path (for logging)
+ *   resolvedFrom:   'v2-source-build',
+ * }>}
+ * @throws {Error} when the cloned manifest lacks a `build` section, when
+ *                 CMake is missing, or when the compile itself fails.
+ */
+async function buildFromSource(args) {
+  const {
+    name, version,
+    profile, profileName, profileHash,
+    gitRepo, gitSha,
+    destZipPath,
+    packageSources,
+    platform,
+    timeoutMs,
+    buildMode = 'missing',
+    configs,
+    authOptions = {},
+  } = args;
+
+  if (!gitRepo || !gitSha) {
+    throw new Error(
+      `Cannot source-build ${name}@${version}: the registry has no git metadata ` +
+      `(gitRepo/gitSha) for this version. Publisher must re-publish with git info.`,
+    );
+  }
+
+  const paths = getBuildPaths({ name, version, profileHash });
+
+  // ── 1. Cache hit? Short-circuit to copy and return ──────────────────────
+  if (hasValidArtefact({ name, version, profileHash })) {
+    log.info(`  source-build cache hit: ${paths.artefact}`);
+    fs.copyFileSync(paths.artefact, destZipPath);
+    const contentSha256 = sha256Of(paths.artefact);
+    return {
+      found:        true,
+      contentSha256,
+      profileHash,
+      gitSha,
+      gitRepo,
+      source:       paths.root,
+      resolvedFrom: 'v2-source-build',
+    };
+  }
+
+  log.info(`  source-build starting: ${name}@${version} [${profileHash}]`);
+  log.info(`  cache dir: ${paths.root}`);
+
+  // ── 2. Clone source ─────────────────────────────────────────────────────
+  fs.mkdirSync(paths.root, { recursive: true });
+  await cloneAtSha({ gitRepo, gitSha, destDir: paths.src });
+
+  // ── 3. Read cloned manifest + recipe ────────────────────────────────────
+  const manifestPath = path.join(paths.src, 'wyvrn.json');
+  if (!fs.existsSync(manifestPath)) {
+    throw new Error(
+      `Cloned source does not contain wyvrn.json at ${manifestPath}. ` +
+      `${name}@${version} cannot be source-built.`,
+    );
+  }
+  const clonedManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+  if (!hasBuildRecipe(clonedManifest)) {
+    throw new Error(
+      `${name}@${version} has no "build" section in its wyvrn.json at ${gitSha.slice(0, 12)}. ` +
+      `Source-build is opt-in — the publisher must add a build recipe.`,
+    );
+  }
+  const recipe = normalizeRecipe(clonedManifest.build);
+
+  // ── 4. Recursively resolve + download this package's OWN deps ──────────
+  // Lazy-require: downloadDependencies is where we were called from, so
+  // pulling it in at module-load would be a circular import.
+  // eslint-disable-next-line global-require
+  const { downloadDependencies } = require('../download');
+
+  const rawPackageDeps = normalizeDependencies(
+    clonedManifest.dependencies ?? clonedManifest.Dependencies,
+  );
+  const depsForResolution = Object.fromEntries(
+    Object.entries(rawPackageDeps).map(([n, d]) => [n, d.version]),
+  );
+
+  log.info(`  resolving ${Object.keys(depsForResolution).length} transitive dep(s)`);
+  const resolvedDeps = await resolveDependencies(
+    depsForResolution, packageSources, platform, fetch, new Map(),
+  );
+
+  // Build an enriched deps map (profileHash per dep based on overrides).
+  const enrichedDeps = new Map();
+  for (const [depName, depVersion] of resolvedDeps) {
+    const depOverrides    = rawPackageDeps[depName]?.settings ?? {};
+    const effectiveProfile = mergeProfile(profile, depOverrides);
+    const depProfileHash   = hashProfile(effectiveProfile);
+    enrichedDeps.set(depName, {
+      version:     depVersion,
+      profileHash: depProfileHash,
+      profile:     effectiveProfile,
+    });
+  }
+
+  if (enrichedDeps.size > 0) {
+    await downloadDependencies(
+      enrichedDeps, packageSources, platform, paths.internal,
+      fetch, Math.max(30, Math.round(timeoutMs / 1000)),
+      { ...authOptions, buildMode },
+    );
+  } else {
+    fs.mkdirSync(paths.internal, { recursive: true });
+  }
+
+  // ── 5. Generate the CMake toolchain into wyvrn_internal/ ─────────────────
+  const { toolchain } = generateToolchain({
+    destDir:     paths.internal,
+    profile,
+    profileName,
+    profileHash,
+    packageNames: [...resolvedDeps.keys()].sort(),
+  });
+
+  // ── 6. Configure + build + install ──────────────────────────────────────
+  // Caller can override recipe.configs (e.g. for CI that only wants Release);
+  // otherwise the recipe's own default of all four CMake configurations is
+  // respected.
+  const effectiveConfigs = (configs && configs.length > 0) ? configs : recipe.configs;
+  log.info(`  building ${name}@${version} (configs: ${effectiveConfigs.join(', ')})`);
+  await configureBuildInstall({
+    recipe,
+    srcRoot:       paths.src,
+    buildPaths:    { build: paths.build, install: paths.install },
+    toolchainFile: toolchain,
+    configs:       effectiveConfigs,
+    profile,
+  });
+
+  // ── 7. Zip install dir → wyvrn.zip ──────────────────────────────────────
+  if (!fs.existsSync(paths.install) || fs.readdirSync(paths.install).length === 0) {
+    throw new Error(
+      `CMake install produced no files at ${paths.install}. ` +
+      `Check the package's install() rules in its CMakeLists.txt.`,
+    );
+  }
+  log.info(`  zipping install dir → ${paths.artefact}`);
+  const zip = new AdmZip();
+  addFolder(zip, paths.install, paths.install);
+  zip.writeZip(paths.artefact);
+
+  const contentSha256 = sha256Of(paths.artefact);
+  log.info(`  source-build SHA256: ${contentSha256.slice(0, 16)}...`);
+
+  // ── 8. Hand off to the caller's extraction flow ─────────────────────────
+  fs.copyFileSync(paths.artefact, destZipPath);
+
+  return {
+    found:        true,
+    contentSha256,
+    profileHash,
+    gitSha,
+    gitRepo,
+    source:       paths.root,
+    resolvedFrom: 'v2-source-build',
+  };
+}
+
+module.exports = { buildFromSource };

package/src/build/msvc-env.js

@@ -0,0 +1,217 @@
+'use strict';
+
+const fs   = require('fs');
+const path = require('path');
+const { spawn } = require('child_process');
+
+const log = require('../logger');
+
+/**
+ * Auto-activate the MSVC build environment (the moral equivalent of a
+ * "Developer PowerShell for VS 2022") when a source-build on Windows picks
+ * a non-VS generator and `cl.exe` isn't on PATH.
+ *
+ * Without this, Ninja/Make/etc. fail at CMake's compiler-detection step
+ * because MSVC's `cl.exe` lives under the VS install tree and is only put
+ * on PATH by `vcvarsall.bat`. VS generators sidestep this because MSBuild
+ * locates MSVC via VS's own toolset resolution.
+ *
+ * Strategy:
+ *   1. Check if cl.exe is already on PATH → return null (nothing to do).
+ *   2. Locate the VS installation via `vswhere.exe -latest`.
+ *   3. Run `vcvarsall.bat <arch>` in `cmd /s /c` and capture `set` output.
+ *   4. Parse KEY=VALUE lines, return them as a shallow-merged env object
+ *      suitable for `child_process.spawn({ env })`.
+ *
+ * Returns `null` when nothing needs to be done OR when VS isn't found —
+ * callers fall back to the ambient environment and CMake produces its own
+ * error if that turns out to be missing a compiler.
+ */
+
+const VSWHERE_DEFAULT = path.join(
+  process.env['ProgramFiles(x86)'] || 'C:\\Program Files (x86)',
+  'Microsoft Visual Studio', 'Installer', 'vswhere.exe',
+);
+
+function tryCapture(cmd, args, opts = {}) {
+  return new Promise((resolve) => {
+    const proc = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'ignore'], ...opts });
+    let out = '';
+    proc.stdout.on('data', (c) => { out += c.toString(); });
+    proc.on('error', () => resolve(null));
+    proc.on('close', (code) => resolve(code === 0 ? out : null));
+  });
+}
+
+/**
+ * @returns {Promise<boolean>}
+ */
+async function isClOnPath() {
+  if (process.platform !== 'win32') return false;
+  // `where cl` is the Windows-native PATH probe.
+  return new Promise((resolve) => {
+    const proc = spawn('where', ['cl'], { stdio: 'ignore' });
+    proc.on('error', () => resolve(false));
+    proc.on('close', (code) => resolve(code === 0));
+  });
+}
+
+/**
+ * Return the latest VS installation path via vswhere, or null if not found.
+ * @returns {Promise<string|null>}
+ */
+async function findVsInstall() {
+  if (!fs.existsSync(VSWHERE_DEFAULT)) return null;
+  const out = await tryCapture(VSWHERE_DEFAULT, [
+    '-latest',
+    '-requires', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64',
+    '-property', 'installationPath',
+  ]);
+  return out ? out.trim() : null;
+}
+
+function findVcvarsall(vsInstallPath) {
+  const p = path.join(vsInstallPath, 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat');
+  return fs.existsSync(p) ? p : null;
+}
+
+/**
+ * Map a wyvrnpm profile arch to the argument `vcvarsall.bat` expects.
+ */
+function mapArch(profileArch) {
+  switch (profileArch) {
+    case 'x86_64': return 'x64';
+    case 'x86':    return 'x86';
+    case 'armv8':  return 'arm64';
+    case 'armv7':  return 'arm';
+    default:       return 'x64';
+  }
+}
+
+/**
+ * Capture the environment produced by running vcvarsall.bat.
+ *
+ * Strategy: shell out to `cmd`, run `vcvarsall.bat <arch>`, then print a
+ * marker followed by the post-vcvars environment (`set` dumps KEY=VALUE
+ * lines for every env var). Everything before the marker is vcvarsall's
+ * own output; everything after is the env we want.
+ *
+ * Uses `shell: true` so Node hands the full command line to `cmd.exe`
+ * verbatim — avoids the usual quoting pitfalls when the VS install path
+ * contains spaces.
+ *
+ * @param {string} vcvarsallPath
+ * @param {string} archArg
+ * @returns {Promise<object|null>}
+ */
+async function captureVcvarsEnv(vcvarsallPath, archArg) {
+  const MARKER = '___WYVRN_ENV_MARKER___';
+  // `call` is required so the batch file returns control instead of
+  // `exit`-ing the cmd shell; otherwise the `echo` + `set` after it
+  // never run.
+  const cmdLine =
+    `call "${vcvarsallPath}" ${archArg} ` +
+    `&& echo ${MARKER} ` +
+    `&& set`;
+
+  return new Promise((resolve) => {
+    const proc = spawn(cmdLine, {
+      shell: true,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    let stdout = '';
+    let stderr = '';
+    proc.stdout.on('data', (c) => { stdout += c.toString(); });
+    proc.stderr.on('data', (c) => { stderr += c.toString(); });
+    proc.on('error', (err) => {
+      log.warn(`  vcvarsall spawn error: ${err.message}`);
+      resolve(null);
+    });
+    proc.on('close', (code) => {
+      if (code !== 0) {
+        const tail = (stderr || stdout).trim().split(/\r?\n/).slice(-5).join(' | ');
+        log.warn(`  vcvarsall exited ${code}; last lines: ${tail}`);
+        resolve(null);
+        return;
+      }
+      const markerIdx = stdout.indexOf(MARKER);
+      if (markerIdx === -1) {
+        const tail = stdout.trim().split(/\r?\n/).slice(-5).join(' | ');
+        log.warn(`  vcvarsall marker not found; last stdout lines: ${tail}`);
+        resolve(null);
+        return;
+      }
+      // Everything after the marker line is the env dump.
+      const envDump = stdout.slice(markerIdx + MARKER.length);
+      const env = { ...process.env };
+      let matched = 0;
+      for (const line of envDump.split(/\r?\n/)) {
+        const m = line.match(/^([A-Za-z_][A-Za-z0-9_()]*)=(.*)$/);
+        if (m) { env[m[1]] = m[2]; matched += 1; }
+      }
+      if (matched < 5 || !env.VSINSTALLDIR) {
+        log.warn(
+          `  vcvarsall env dump looks incomplete ` +
+          `(${matched} vars, VSINSTALLDIR=${env.VSINSTALLDIR ? 'set' : 'MISSING'})`,
+        );
+        resolve(null);
+        return;
+      }
+      resolve(env);
+    });
+  });
+}
+
+/**
+ * Compute a build-environment override for the given profile, or return
+ * null to indicate "use the ambient environment". Callers pass the result
+ * as `spawn(..., { env })`.
+ *
+ * Only activates when:
+ *   - Running on Windows
+ *   - Profile says compiler=msvc
+ *   - `cl.exe` is NOT already on PATH (so we're outside a Dev shell)
+ *   - `skipForGenerator` does not start with "Visual Studio" (VS generators
+ *     don't need this — MSBuild resolves MSVC itself)
+ *
+ * @param {{
+ *   profile:             { compiler?: string, arch?: string },
+ *   skipForGenerator?:   string|null,
+ * }} args
+ * @returns {Promise<object|null>}
+ */
+async function loadMsvcEnv({ profile, skipForGenerator }) {
+  if (process.platform !== 'win32')       return null;
+  if (!profile || profile.compiler !== 'msvc') return null;
+  if (skipForGenerator && skipForGenerator.startsWith('Visual Studio')) return null;
+  if (await isClOnPath())                 return null;
+
+  const vsPath = await findVsInstall();
+  if (!vsPath) {
+    log.warn('  MSVC profile requested but vswhere/VS not found — CMake will error if cl.exe is missing');
+    return null;
+  }
+  const vcvarsall = findVcvarsall(vsPath);
+  if (!vcvarsall) {
+    log.warn(`  MSVC profile requested but vcvarsall.bat not found under ${vsPath}`);
+    return null;
+  }
+
+  const archArg = mapArch(profile.arch);
+  log.info(`  activating MSVC build env: vcvarsall.bat ${archArg} (${vsPath})`);
+  const env = await captureVcvarsEnv(vcvarsall, archArg);
+  if (!env) {
+    log.warn(`  vcvarsall.bat ${archArg} failed — CMake will error if cl.exe is missing`);
+    return null;
+  }
+  return env;
+}
+
+module.exports = {
+  loadMsvcEnv,
+  // Exported for tests
+  isClOnPath,
+  findVsInstall,
+  findVcvarsall,
+  mapArch,
+};

package/src/build/recipe.js

@@ -0,0 +1,129 @@
+'use strict';
+
+/**
+ * The `build` section of a source-repo `wyvrn.json` tells wyvrnpm how to
+ * build the package from source when `--build=missing` fires. Only
+ * `system: "cmake"` is supported in phase 3.
+ *
+ * Example — single generator (strict; CMake errors if Ninja isn't installed):
+ *   "build": {
+ *     "system":       "cmake",
+ *     "generator":    "Ninja",
+ *     "configure":    ["-DZLIB_BUILD_EXAMPLES=OFF"],
+ *     "buildArgs":    ["--parallel"],
+ *     "installDir":   "install",
+ *     "sourceSubdir": "."
+ *   }
+ *
+ * Example — generator fallback chain (tried in order, first available wins;
+ *           falls through to CMake's default if none are present):
+ *   "build": {
+ *     "generator": ["Ninja", "Visual Studio 17 2022", "Visual Studio 16 2019"]
+ *   }
+ */
+
+const DEFAULT_RECIPE = Object.freeze({
+  system:        'cmake',
+  // Null / empty array → let CMake pick its default
+  // (Visual Studio on Windows with VS installed, Unix Makefiles on Linux/macOS).
+  generators:    Object.freeze([]),
+  // All four CMake configurations are built and installed by default so
+  // downstream consumers can link against whichever they need. Restrict via
+  // the recipe (e.g. `"configs": ["Release"]`) to cut source-build time.
+  configs:       Object.freeze(['Debug', 'Release', 'RelWithDebInfo', 'MinSizeRel']),
+  configure:     Object.freeze([]),
+  buildArgs:     Object.freeze([]),
+  installDir:    'install',                 // relative to binary dir; becomes CMAKE_INSTALL_PREFIX
+  sourceSubdir:  '.',                       // where CMakeLists.txt lives relative to the clone root
+  requiredTools: Object.freeze([]),
+});
+
+const VALID_CMAKE_CONFIGS = new Set(['Debug', 'Release', 'RelWithDebInfo', 'MinSizeRel']);
+
+const SUPPORTED_SYSTEMS = new Set(['cmake']);
+
+/**
+ * Normalise the `build` section of a manifest into a concrete, complete
+ * recipe. Missing fields fall back to `DEFAULT_RECIPE`.
+ *
+ * `generator` accepts either a string (strict — single generator) or an
+ * array of strings (fallback chain — tried in order; first available wins).
+ * Both forms produce a `generators: string[]` field in the normalised recipe.
+ *
+ * @param {object|null|undefined} rawBuild
+ * @returns {{ system: string, generators: string[], configure: string[],
+ *            buildArgs: string[], installDir: string, sourceSubdir: string,
+ *            requiredTools: string[] }}
+ * @throws {Error} when `system` is set but not supported
+ */
+function normalizeRecipe(rawBuild) {
+  const b = rawBuild ?? {};
+  const system = b.system ?? DEFAULT_RECIPE.system;
+  if (!SUPPORTED_SYSTEMS.has(system)) {
+    throw new Error(
+      `build.system "${system}" is not supported. ` +
+      `Supported: ${[...SUPPORTED_SYSTEMS].join(', ')}.`,
+    );
+  }
+
+  // Accept generator as string | string[] | null for user convenience;
+  // canonicalise to an array.
+  let generators;
+  if (typeof b.generator === 'string') {
+    generators = b.generator.trim() ? [b.generator] : [];
+  } else if (Array.isArray(b.generator)) {
+    generators = b.generator.filter((g) => typeof g === 'string' && g.trim());
+  } else {
+    generators = [];
+  }
+
+  // Drop empty-string entries so a stray `""` in the user's JSON doesn't
+  // become an empty-string argument to CMake / the build tool.
+  const cleanArray = (arr) =>
+    Array.isArray(arr)
+      ? arr.filter((s) => typeof s === 'string' && s.trim().length > 0)
+      : [];
+
+  // Accept `configs` as string | string[]. Default to all four canonical
+  // CMake configurations. Unknown entries are silently dropped so a typo
+  // doesn't blow up the build loop.
+  let configs;
+  if (typeof b.configs === 'string') {
+    configs = [b.configs];
+  } else if (Array.isArray(b.configs)) {
+    configs = b.configs.filter((c) => typeof c === 'string' && c.trim());
+  } else {
+    configs = DEFAULT_RECIPE.configs.slice();
+  }
+  configs = configs.filter((c) => VALID_CMAKE_CONFIGS.has(c));
+  if (configs.length === 0) configs = DEFAULT_RECIPE.configs.slice();
+
+  return {
+    system,
+    generators,
+    configs,
+    configure:     cleanArray(b.configure),
+    buildArgs:     cleanArray(b.buildArgs),
+    installDir:    b.installDir    ?? DEFAULT_RECIPE.installDir,
+    sourceSubdir:  b.sourceSubdir  ?? DEFAULT_RECIPE.sourceSubdir,
+    requiredTools: cleanArray(b.requiredTools),
+  };
+}
+
+/**
+ * Returns true if a manifest has enough info for source-build — i.e. the
+ * `build` section is present (even empty — defaults cover the rest).
+ * @param {object} manifest
+ * @returns {boolean}
+ */
+function hasBuildRecipe(manifest) {
+  return !!manifest && typeof manifest.build === 'object' && manifest.build !== null;
+}
+
+module.exports = {
+  normalizeRecipe,
+  hasBuildRecipe,
+  DEFAULT_RECIPE,
+  SUPPORTED_SYSTEMS,
+  VALID_CMAKE_CONFIGS,
+};