zx-bulk-release 1.26.2 → 2.0.0

package/CHANGELOG.md

@@ -1,3 +1,11 @@
+## [2.0.0](https://github.com/semrel-extra/zx-bulk-release/compare/v1.26.2...v2.0.0) (2023-03-21)
+
+### Features
+* feat: replace sequent flow with concurrent ([ec35d7e](https://github.com/semrel-extra/zx-bulk-release/commit/ec35d7e337e64369f0423b71008345177f3ac949))
+
+### BREAKING CHANGES
+* concurrent flow might cause throttling issues ([ec35d7e](https://github.com/semrel-extra/zx-bulk-release/commit/ec35d7e337e64369f0423b71008345177f3ac949))
+
 ## [1.26.2](https://github.com/semrel-extra/zx-bulk-release/compare/v1.26.1...v1.26.2) (2023-03-21)
 
 ### Fixes & improvements

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zx-bulk-release",
-  "version": "1.26.2",
+  "version": "2.0.0",
   "description": "zx-based alternative for multi-semantic-release",
   "type": "module",
   "exports": {
@@ -23,6 +23,7 @@
   "dependencies": {
     "@semrel-extra/topo": "^1.6.0",
     "cosmiconfig": "^8.1.3",
+    "queuefy": "^1.2.1",
     "zx-extra": "^2.5.4"
   },
   "devDependencies": {

package/src/main/js/build.js

@@ -1,18 +1,20 @@
 import {traverseDeps} from './deps.js'
 import {fetchPkg} from './npm.js'
-import {runHook} from './processor.js'
+import {runCmd} from './processor.js'
 
-export const build = async (pkg, packages) => {
+export const build = async (pkg, packages, run = runCmd) => {
   if (pkg.built) return true
 
   await traverseDeps(pkg, packages, async (_, {pkg}) => build(pkg, packages))
 
   const {config} = pkg
 
-  if (pkg.changes.length === 0 && config.npmFetch) await fetchPkg(pkg)
+  if (pkg.changes.length === 0 && config.npmFetch) {
+    await fetchPkg(pkg)
+  }
 
   if (!pkg.fetched && config.buildCmd) {
-    await runHook(pkg, 'buildCmd')
+    await run(pkg, 'buildCmd')
   }
 
   pkg.built = true

package/src/main/js/index.js

@@ -1 +1 @@
-export {run} from './processor.js'
+export {run} from './processor.js'

package/src/main/js/processor.js

@@ -1,16 +1,19 @@
 import os from 'node:os'
 import {$, fs, within} from 'zx-extra'
 import {log} from './log.js'
-import {topo} from './topo.js'
+import {topo, traverse} from './topo.js'
 import {contextify} from './contextify.js'
 import {analyze} from './analyze.js'
 import {build} from './build.js'
 import {publish} from './publish.js'
 import {createState} from './state.js'
-import {tpl} from "./util.js";
+import {tpl} from './util.js'
+import {queuefy} from 'queuefy'
 
 export const run = async ({cwd = process.cwd(), env, flags = {}, concurrency = os.cpus().length} = {}) => within(async () => {
   const state = createState({file: flags.report})
+  const _runCmd = queuefy(runCmd, concurrency)
+
   $.state = state
   $.env = {...process.env, ...env}
   $.verbose = !!(flags.debug || $.env.DEBUG ) || $.verbose
@@ -18,16 +21,14 @@ export const run = async ({cwd = process.cwd(), env, flags = {}, concurrency = o
   log()('zx-bulk-release')
 
   try {
-    const {packages, queue, root} = await topo({cwd, flags})
+    const {packages, queue, root, sources, next, prev, nodes} = await topo({cwd, flags})
     log()('queue:', queue)
 
     state.setQueue(queue, packages)
-    state.setStatus('pending')
-
-    for (let name of queue) {
-      const pkg = packages[name]
 
+    await traverse({nodes, prev, async cb(name) {
       state.setStatus('analyzing', name)
+      const pkg = packages[name]
       await contextify(pkg, packages, root)
       await analyze(pkg, packages)
       state.set('config', pkg.config, name)
@@ -35,25 +36,31 @@ export const run = async ({cwd = process.cwd(), env, flags = {}, concurrency = o
       state.set('prevVersion', pkg.latest.tag?.version || pkg.manifest.version, name)
       state.set('releaseType', pkg.releaseType, name)
       state.set('tag', pkg.tag, name)
+    }})
+
+    state.setStatus('pending')
+
+    await traverse({nodes, prev, async cb(name) {
+      const pkg = packages[name]
 
       if (!pkg.releaseType) {
         state.setStatus('skipped', name)
-        continue
+        return
       }
 
       state.setStatus('building', name)
-      await build(pkg, packages)
+      await build(pkg, packages, _runCmd)
 
       if (flags.dryRun) {
         state.setStatus('success', name)
-        continue
+        return
       }
 
       state.setStatus('publishing', name)
-      await publish(pkg)
+      await publish(pkg, _runCmd)
 
       state.setStatus('success', name)
-    }
+    }})
   } catch (e) {
     log({level: 'error'})(e)
     state.set('error', e)
@@ -64,7 +71,7 @@ export const run = async ({cwd = process.cwd(), env, flags = {}, concurrency = o
   log()('Great success!')
 })
 
-export const runHook = async (pkg, name) => {
+export const runCmd = async (pkg, name) => {
   const cmd = tpl(pkg.config[name], {...pkg, ...pkg.context})
 
   if (cmd) {

package/src/main/js/publish.js

@@ -3,11 +3,11 @@ import {formatTag, getLatestTag, pushTag} from './tag.js'
 import {push, fetch, parseRepo} from './repo.js'
 import {parseEnv} from './config.js'
 import {fetchManifest, npmPublish} from './npm.js'
-import {restJoin} from './util.js'
-import {runHook} from './processor.js'
+import {msgJoin} from './util.js'
+import {runCmd} from './processor.js'
 import {log} from './log.js'
 
-export const publish = async (pkg) => {
+export const publish = async (pkg, run = runCmd) => {
   await fs.writeJson(pkg.manifestPath, pkg.manifest, {spaces: 2})
   await pushTag(pkg)
   await pushMeta(pkg)
@@ -15,7 +15,7 @@ export const publish = async (pkg) => {
   await npmPublish(pkg)
   await ghRelease(pkg)
   await ghPages(pkg)
-  await runHook(pkg, 'publishCmd')
+  await run(pkg, 'publishCmd')
 }
 
 export const pushMeta = async (pkg) => {
@@ -70,7 +70,7 @@ const pushChangelog = async (pkg) => {
     ? opts.split(' ')
     : [opts.branch, opts.file, opts.msg]
   const _cwd = await fetch({cwd: pkg.absPath, branch})
-  const msg = restJoin(_msg, pkg, 'chore: update changelog ${{name}}')
+  const msg = msgJoin(_msg, pkg, 'chore: update changelog ${{name}}')
   const releaseNotes = await formatReleaseNotes(pkg)
 
   await $.o({cwd: _cwd})`echo ${releaseNotes}"\n$(cat ./${file})" > ./${file}`
@@ -105,7 +105,7 @@ const ghPages = async (pkg) => {
   const [branch = 'gh-pages', from = 'docs', to = '.', ..._msg] = typeof opts === 'string'
     ? opts.split(' ')
     : [opts.branch, opts.from, opts.to, opts.msg]
-  const msg = restJoin(_msg, pkg, 'docs: update docs ${{name}} ${{version}}')
+  const msg = msgJoin(_msg, pkg, 'docs: update docs ${{name}} ${{version}}')
 
   log({pkg})(`publish docs to ${branch}`)
 

package/src/main/js/repo.js

@@ -1,6 +1,7 @@
 import {$, ctx, fs, path, tempy, copy} from 'zx-extra'
 import {parseEnv} from './config.js'
 import {log} from './log.js'
+import {keyByValue} from './util.js'
 
 const branches = {}
 export const fetch = async ({cwd: _cwd, branch, origin: _origin}) => ctx(async ($) => {
@@ -25,28 +26,39 @@ export const fetch = async ({cwd: _cwd, branch, origin: _origin}) => ctx(async (
 })
 
 export const push = async ({cwd, from, to, branch, origin, msg, ignoreFiles, files = []}) => ctx(async ($) => {
-  const _cwd = await fetch({cwd, branch, origin})
-  const {gitCommitterEmail, gitCommitterName} = parseEnv($.env)
-
-  for (let {relpath, contents} of files) {
-    const _contents = typeof contents === 'string' ? contents : JSON.stringify(contents, null, 2)
-    await fs.outputFile(path.resolve(_cwd, to, relpath), _contents)
-  }
-  if (from) await copy({baseFrom: cwd, from, baseTo: _cwd, to, ignoreFiles, cwd})
-
-  $.cwd = _cwd
-
-  await $`git config user.name ${gitCommitterName}`
-  await $`git config user.email ${gitCommitterEmail}`
-  await $`git add .`
-  try {
-    await $`git commit -m ${msg}`
-  } catch {
-    log({level: 'warn'})(`no changes to commit to ${branch}`)
-    return
+  let retries = 3
+  let _cwd
+
+  while (retries > 0) {
+    try {
+      const {gitCommitterEmail, gitCommitterName} = parseEnv($.env)
+      _cwd = await fetch({cwd, branch, origin})
+
+      for (let {relpath, contents} of files) {
+        const _contents = typeof contents === 'string' ? contents : JSON.stringify(contents, null, 2)
+        await fs.outputFile(path.resolve(_cwd, to, relpath), _contents)
+      }
+      if (from) await copy({baseFrom: cwd, from, baseTo: _cwd, to, ignoreFiles, cwd})
+
+      $.cwd = _cwd
+
+      await $`git config user.name ${gitCommitterName}`
+      await $`git config user.email ${gitCommitterEmail}`
+      await $`git add .`
+      try {
+        await $`git commit -m ${msg}`
+      } catch {
+        log({level: 'warn'})(`no changes to commit to ${branch}`)
+        return
+      }
+
+      return await $.raw`git push origin HEAD:refs/heads/${branch}`
+    } catch (e) {
+      retries -= 1
+      branches[keyByValue(branches, _cwd)] = null
+      console.warn('git push failed', 'retries left', retries, e)
+    }
   }
-
-  await $.raw`git push origin HEAD:refs/heads/${branch}`
 })
 
 const repos = {}

package/src/main/js/topo.js

@@ -1,4 +1,5 @@
 import {topo as _topo} from '@semrel-extra/topo'
+import {getPromise} from './util.js'
 
 export const topo = async ({flags = {}, cwd} = {}) => {
   const ignore = typeof flags.ignore === 'string'
@@ -14,4 +15,17 @@ export const topo = async ({flags = {}, cwd} = {}) => {
       !ignore.includes(name)
 
   return _topo({cwd, filter})
-}
+}
+
+export const traverse = async ({nodes, prev, cb}) => {
+  const waitings = nodes.reduce((acc, node) => {
+    acc[node] = getPromise()
+    return acc
+  }, {})
+
+  await Promise.all(nodes.map(async (name) => {
+    await Promise.all((prev.get(name) || []).map((p) => waitings[p].promise))
+    await cb(name)
+    waitings[name].resolve(true)
+  }))
+}

package/src/main/js/util.js

@@ -1,5 +1,3 @@
-import {$, fs} from 'zx-extra'
-
 export const tpl = (str, context) =>
   str?.replace(/\$\{\{\s*([.a-z0-9]+)\s*}}/gi, (matched, key) => get(context, key) ?? '')
 
@@ -30,4 +28,19 @@ export const set = (obj, path, value) => {
     return result
 }
 
-export const restJoin = (rest, context, def) => tpl(rest.filter(Boolean).join(' ') || def, context)
+export const msgJoin = (rest, context, def) => tpl(rest.filter(Boolean).join(' ') || def, context)
+
+export const getPromise = () => {
+    let resolve, reject
+    const promise = new Promise((...args) => {
+        [resolve, reject] = args
+    })
+
+    return {
+        resolve,
+        reject,
+        promise,
+    }
+}
+
+export const keyByValue = (obj, value) => Object.keys(obj).find((key) => obj[key] === value)