zx-bulk-release 1.26.1 → 2.0.0

package/CHANGELOG.md

@@ -1,3 +1,16 @@
+## [2.0.0](https://github.com/semrel-extra/zx-bulk-release/compare/v1.26.2...v2.0.0) (2023-03-21)
+
+### Features
+* feat: replace sequent flow with concurrent ([ec35d7e](https://github.com/semrel-extra/zx-bulk-release/commit/ec35d7e337e64369f0423b71008345177f3ac949))
+
+### BREAKING CHANGES
+* concurrent flow might cause throttling issues ([ec35d7e](https://github.com/semrel-extra/zx-bulk-release/commit/ec35d7e337e64369f0423b71008345177f3ac949))
+
+## [1.26.2](https://github.com/semrel-extra/zx-bulk-release/compare/v1.26.1...v1.26.2) (2023-03-21)
+
+### Fixes & improvements
+* refactor: separate processor layer ([dae6b96](https://github.com/semrel-extra/zx-bulk-release/commit/dae6b9682cd7d283e4f776c69f30c5f50fe9fa41))
+
 ## [1.26.1](https://github.com/semrel-extra/zx-bulk-release/compare/v1.26.0...v1.26.1) (2023-02-08)
 
 ### Fixes & improvements

package/README.md

@@ -278,7 +278,7 @@ Release process state is reported to the console and to a file if `--report` fla
   events: [
     {msg: ['zx-bulk-release'], scope:'~', date: 1665839585488, level: 'info'},
     {msg: ['queue:',['a','b']], scope:'~', date: 1665839585493, level: 'info'},
-    {msg: ["run buildCmd 'yarn && yarn build && yarn test'"], scope: 'a', date:1665839585719, level:'info'},
+    {msg: ["run buildCmd 'yarn && yarn build && yarn test'"], scope: 'a', date: 1665839585719, level:'info'},
     // ...
   ]
 }
@@ -299,6 +299,8 @@ Release process state is reported to the console and to a file if `--report` fla
 * [vercel/turborepo](https://github.com/vercel/turborepo)
 * [lerna/lerna](https://github.com/lerna/lerna)
 * [nrwl/nx](https://github.com/nrwl/nx)
+* [moonrepo/moon](https://github.com/moonrepo/moon)
+* [ojkelly/yarn.build](https://github.com/ojkelly/yarn.build)
 
 ## License 
 [MIT](./LICENSE)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zx-bulk-release",
-  "version": "1.26.1",
+  "version": "2.0.0",
   "description": "zx-based alternative for multi-semantic-release",
   "type": "module",
   "exports": {
@@ -21,14 +21,15 @@
     "docs": "mkdir -p docs && cp ./README.md ./docs/README.md"
   },
   "dependencies": {
-    "@semrel-extra/topo": "^1.4.4",
-    "cosmiconfig": "^8.0.0",
-    "zx-extra": "^2.5.3"
+    "@semrel-extra/topo": "^1.6.0",
+    "cosmiconfig": "^8.1.3",
+    "queuefy": "^1.2.1",
+    "zx-extra": "^2.5.4"
   },
   "devDependencies": {
-    "c8": "^7.12.0",
+    "c8": "^7.13.0",
     "uvu": "^0.5.6",
-    "verdaccio": "^5.19.1"
+    "verdaccio": "^5.22.1"
   },
   "publishConfig": {
     "access": "public"

package/src/main/js/analyze.js

@@ -1,7 +1,7 @@
 import {ctx, semver} from 'zx-extra'
 import {updateDeps} from './deps.js'
 import {formatTag} from './tag.js';
-import {log} from './util.js'
+import {log} from './log.js'
 
 export const analyze = async (pkg, packages) => {
   const semanticChanges = await getSemanticChanges(pkg.absPath, pkg.latest.tag?.ref)

package/src/main/js/build.js

@@ -1,18 +1,20 @@
 import {traverseDeps} from './deps.js'
 import {fetchPkg} from './npm.js'
-import {runHook} from './util.js'
+import {runCmd} from './processor.js'
 
-export const build = async (pkg, packages) => {
+export const build = async (pkg, packages, run = runCmd) => {
   if (pkg.built) return true
 
   await traverseDeps(pkg, packages, async (_, {pkg}) => build(pkg, packages))
 
   const {config} = pkg
 
-  if (pkg.changes.length === 0 && config.npmFetch) await fetchPkg(pkg)
+  if (pkg.changes.length === 0 && config.npmFetch) {
+    await fetchPkg(pkg)
+  }
 
   if (!pkg.fetched && config.buildCmd) {
-    await runHook(pkg, 'buildCmd')
+    await run(pkg, 'buildCmd')
   }
 
   pkg.built = true

package/src/main/js/index.js

@@ -1,62 +1 @@
-import {within, $} from 'zx-extra'
-import {analyze} from './analyze.js'
-import {publish} from './publish.js'
-import {build} from './build.js'
-import {contextify} from './contextify.js'
-import {topo} from './topo.js'
-import {createReporter, log} from './util.js';
-
-export {getLatestTaggedVersion} from './tag.js'
-
-export const run = async ({cwd = process.cwd(), env, flags = {}} = {}) => within(async () => {
-  const reporter = $.r = createReporter(flags.report)
-  $.env = {...process.env, ...env}
-  $.verbose = !!(flags.debug || $.env.DEBUG ) || $.verbose
-  log()('zx-bulk-release')
-
-  try {
-  const {packages, queue, root} = await topo({cwd, flags})
-  log()('queue:', queue)
-
-  reporter.setQueue(queue, packages)
-  reporter.setStatus('pending')
-
-  for (let name of queue) {
-    const pkg = packages[name]
-
-    reporter.setStatus('analyzing', name)
-    await contextify(pkg, packages, root)
-    await analyze(pkg, packages)
-    reporter.setState('config', pkg.config, name)
-    reporter.setState('version', pkg.version, name)
-    reporter.setState('prevVersion', pkg.latest.tag?.version || pkg.manifest.version, name)
-    reporter.setState('releaseType', pkg.releaseType, name)
-    reporter.setState('tag', pkg.tag, name)
-
-    if (!pkg.releaseType) {
-      reporter.setStatus('skipped', name)
-      continue
-    }
-
-    reporter.setStatus('building', name)
-    await build(pkg, packages)
-
-    if (flags.dryRun) {
-      reporter.setStatus('success', name)
-      continue
-    }
-
-    reporter.setStatus('publishing', name)
-    await publish(pkg)
-
-    reporter.setStatus('success', name)
-  }
-  } catch (e) {
-    log({level: 'error'})(e)
-    reporter.setState('error', e)
-    reporter.setStatus('failure')
-    throw e
-  }
-  reporter.setStatus('success')
-  log()('Great success!')
-})
+export {run} from './processor.js'

package/src/main/js/log.js

@@ -0,0 +1,9 @@
+import {$} from 'zx-extra'
+
+export const log = (ctx) => {
+  if ($.state) {
+    return $.state.log(ctx)
+  }
+
+  return console.log
+}

package/src/main/js/npm.js

@@ -1,5 +1,5 @@
 import {parseEnv} from './config.js'
-import {log} from './util.js'
+import {log} from './log.js'
 import {$, ctx, fs, path, tempy, copy, INI, fetch} from 'zx-extra'
 
 export const fetchPkg = async (pkg, {env = $.env} = {}) => {

package/src/main/js/processor.js

@@ -0,0 +1,81 @@
+import os from 'node:os'
+import {$, fs, within} from 'zx-extra'
+import {log} from './log.js'
+import {topo, traverse} from './topo.js'
+import {contextify} from './contextify.js'
+import {analyze} from './analyze.js'
+import {build} from './build.js'
+import {publish} from './publish.js'
+import {createState} from './state.js'
+import {tpl} from './util.js'
+import {queuefy} from 'queuefy'
+
+export const run = async ({cwd = process.cwd(), env, flags = {}, concurrency = os.cpus().length} = {}) => within(async () => {
+  const state = createState({file: flags.report})
+  const _runCmd = queuefy(runCmd, concurrency)
+
+  $.state = state
+  $.env = {...process.env, ...env}
+  $.verbose = !!(flags.debug || $.env.DEBUG ) || $.verbose
+
+  log()('zx-bulk-release')
+
+  try {
+    const {packages, queue, root, sources, next, prev, nodes} = await topo({cwd, flags})
+    log()('queue:', queue)
+
+    state.setQueue(queue, packages)
+
+    await traverse({nodes, prev, async cb(name) {
+      state.setStatus('analyzing', name)
+      const pkg = packages[name]
+      await contextify(pkg, packages, root)
+      await analyze(pkg, packages)
+      state.set('config', pkg.config, name)
+      state.set('version', pkg.version, name)
+      state.set('prevVersion', pkg.latest.tag?.version || pkg.manifest.version, name)
+      state.set('releaseType', pkg.releaseType, name)
+      state.set('tag', pkg.tag, name)
+    }})
+
+    state.setStatus('pending')
+
+    await traverse({nodes, prev, async cb(name) {
+      const pkg = packages[name]
+
+      if (!pkg.releaseType) {
+        state.setStatus('skipped', name)
+        return
+      }
+
+      state.setStatus('building', name)
+      await build(pkg, packages, _runCmd)
+
+      if (flags.dryRun) {
+        state.setStatus('success', name)
+        return
+      }
+
+      state.setStatus('publishing', name)
+      await publish(pkg, _runCmd)
+
+      state.setStatus('success', name)
+    }})
+  } catch (e) {
+    log({level: 'error'})(e)
+    state.set('error', e)
+    state.setStatus('failure')
+    throw e
+  }
+  state.setStatus('success')
+  log()('Great success!')
+})
+
+export const runCmd = async (pkg, name) => {
+  const cmd = tpl(pkg.config[name], {...pkg, ...pkg.context})
+
+  if (cmd) {
+    log({pkg})(`run ${name} '${cmd}'`)
+    await $.o({cwd: pkg.absPath, quote: v => v, preferLocal: true})`${cmd}`
+  }
+}

package/src/main/js/publish.js

@@ -1,11 +1,13 @@
-import {formatTag, getLatestTag, pushTag} from './tag.js'
 import {fs, path, $} from 'zx-extra'
+import {formatTag, getLatestTag, pushTag} from './tag.js'
 import {push, fetch, parseRepo} from './repo.js'
 import {parseEnv} from './config.js'
 import {fetchManifest, npmPublish} from './npm.js'
-import {restJoin, runHook, log} from './util.js'
+import {msgJoin} from './util.js'
+import {runCmd} from './processor.js'
+import {log} from './log.js'
 
-export const publish = async (pkg) => {
+export const publish = async (pkg, run = runCmd) => {
   await fs.writeJson(pkg.manifestPath, pkg.manifest, {spaces: 2})
   await pushTag(pkg)
   await pushMeta(pkg)
@@ -13,7 +15,7 @@ export const publish = async (pkg) => {
   await npmPublish(pkg)
   await ghRelease(pkg)
   await ghPages(pkg)
-  await runHook(pkg, 'publishCmd')
+  await run(pkg, 'publishCmd')
 }
 
 export const pushMeta = async (pkg) => {
@@ -68,7 +70,7 @@ const pushChangelog = async (pkg) => {
     ? opts.split(' ')
     : [opts.branch, opts.file, opts.msg]
   const _cwd = await fetch({cwd: pkg.absPath, branch})
-  const msg = restJoin(_msg, pkg, 'chore: update changelog ${{name}}')
+  const msg = msgJoin(_msg, pkg, 'chore: update changelog ${{name}}')
   const releaseNotes = await formatReleaseNotes(pkg)
 
   await $.o({cwd: _cwd})`echo ${releaseNotes}"\n$(cat ./${file})" > ./${file}`
@@ -103,7 +105,7 @@ const ghPages = async (pkg) => {
   const [branch = 'gh-pages', from = 'docs', to = '.', ..._msg] = typeof opts === 'string'
     ? opts.split(' ')
     : [opts.branch, opts.from, opts.to, opts.msg]
-  const msg = restJoin(_msg, pkg, 'docs: update docs ${{name}} ${{version}}')
+  const msg = msgJoin(_msg, pkg, 'docs: update docs ${{name}} ${{version}}')
 
   log({pkg})(`publish docs to ${branch}`)
 

package/src/main/js/repo.js

@@ -1,6 +1,7 @@
 import {$, ctx, fs, path, tempy, copy} from 'zx-extra'
 import {parseEnv} from './config.js'
-import {log} from './util.js'
+import {log} from './log.js'
+import {keyByValue} from './util.js'
 
 const branches = {}
 export const fetch = async ({cwd: _cwd, branch, origin: _origin}) => ctx(async ($) => {
@@ -25,28 +26,39 @@ export const fetch = async ({cwd: _cwd, branch, origin: _origin}) => ctx(async (
 })
 
 export const push = async ({cwd, from, to, branch, origin, msg, ignoreFiles, files = []}) => ctx(async ($) => {
-  const _cwd = await fetch({cwd, branch, origin})
-  const {gitCommitterEmail, gitCommitterName} = parseEnv($.env)
-
-  for (let {relpath, contents} of files) {
-    const _contents = typeof contents === 'string' ? contents : JSON.stringify(contents, null, 2)
-    await fs.outputFile(path.resolve(_cwd, to, relpath), _contents)
-  }
-  if (from) await copy({baseFrom: cwd, from, baseTo: _cwd, to, ignoreFiles, cwd})
-
-  $.cwd = _cwd
-
-  await $`git config user.name ${gitCommitterName}`
-  await $`git config user.email ${gitCommitterEmail}`
-  await $`git add .`
-  try {
-    await $`git commit -m ${msg}`
-  } catch {
-    log({level: 'warn'})(`no changes to commit to ${branch}`)
-    return
+  let retries = 3
+  let _cwd
+
+  while (retries > 0) {
+    try {
+      const {gitCommitterEmail, gitCommitterName} = parseEnv($.env)
+      _cwd = await fetch({cwd, branch, origin})
+
+      for (let {relpath, contents} of files) {
+        const _contents = typeof contents === 'string' ? contents : JSON.stringify(contents, null, 2)
+        await fs.outputFile(path.resolve(_cwd, to, relpath), _contents)
+      }
+      if (from) await copy({baseFrom: cwd, from, baseTo: _cwd, to, ignoreFiles, cwd})
+
+      $.cwd = _cwd
+
+      await $`git config user.name ${gitCommitterName}`
+      await $`git config user.email ${gitCommitterEmail}`
+      await $`git add .`
+      try {
+        await $`git commit -m ${msg}`
+      } catch {
+        log({level: 'warn'})(`no changes to commit to ${branch}`)
+        return
+      }
+
+      return await $.raw`git push origin HEAD:refs/heads/${branch}`
+    } catch (e) {
+      retries -= 1
+      branches[keyByValue(branches, _cwd)] = null
+      console.warn('git push failed', 'retries left', retries, e)
+    }
   }
-
-  await $.raw`git push origin HEAD:refs/heads/${branch}`
 })
 
 const repos = {}
@@ -58,8 +70,8 @@ export const parseRepo = async (_cwd) => {
   const originUrl = await getOrigin(cwd)
   const [, , repoHost, repoName] = originUrl.replace(':', '/').replace(/\.git/, '').match(/.+(@|\/\/)([^/]+)\/(.+)$/) || []
   const repoPublicUrl = `https://${repoHost}/${repoName}`
-  const repoAuthedUrl = ghToken && ghUser && repoHost && repoName ?
-    `https://${ghUser}:${ghToken}@${repoHost}/${repoName}.git`
+  const repoAuthedUrl = ghToken && ghUser && repoHost && repoName
+    ? `https://${ghUser}:${ghToken}@${repoHost}/${repoName}.git`
     : originUrl
 
   repos[cwd] = {

package/src/main/js/state.js

@@ -0,0 +1,56 @@
+import {get, set, tpl} from './util.js'
+import {fs} from 'zx-extra'
+
+export const createState = ({logger = console, file = null} = {}) => ({
+  logger,
+  file,
+  status: 'initial',
+  queue: [],
+  packages: [],
+  events: [],
+  setQueue(queue, packages) {
+    this.queue = queue
+    this.packages = queue.map(name => {
+      const {manifest: {version}, absPath, relPath} = packages[name]
+      return {
+        status: 'initial',
+        name,
+        version,
+        path: absPath,
+        relPath
+      }
+    })
+  },
+  get(key, pkgName) {
+    return get(
+      pkgName ? this.packages.find(({name}) => name === pkgName) : this,
+      key
+    )
+  },
+  set(key, value, pkgName) {
+    set(
+      pkgName ? this.packages.find(({name}) => name === pkgName) : this,
+      key,
+      value
+    )
+  },
+  setStatus(status, name) {
+    this.set('status', status, name)
+    this.save()
+  },
+  getStatus(status, name) {
+    return this.get('status', name)
+  },
+  log(ctx = {}) {
+    return function (...chunks) {
+      const {pkg, scope = pkg?.name || '~', level = 'info'} = ctx
+      const msg = chunks.map(c => typeof c === 'string' ? tpl(c, ctx) : c)
+      const event = {msg, scope, date: Date.now(), level}
+      this.events.push(event)
+      logger[level](`[${scope}]`, ...msg)
+    }.bind(this)
+  },
+  save() {
+    this.file && fs.outputJsonSync(this.file, this)
+  }
+})

package/src/main/js/tag.js

@@ -3,7 +3,7 @@
 import {ctx, semver, $} from 'zx-extra'
 import {Buffer} from 'buffer'
 import {parseEnv} from './config.js'
-import {log} from './util.js'
+import {log} from './log.js'
 
 export const pushTag = (pkg) => ctx(async ($) => {
   const {absPath: cwd, name, version} = pkg

package/src/main/js/topo.js

@@ -1,4 +1,5 @@
 import {topo as _topo} from '@semrel-extra/topo'
+import {getPromise} from './util.js'
 
 export const topo = async ({flags = {}, cwd} = {}) => {
   const ignore = typeof flags.ignore === 'string'
@@ -14,4 +15,17 @@ export const topo = async ({flags = {}, cwd} = {}) => {
       !ignore.includes(name)
 
   return _topo({cwd, filter})
-}
+}
+
+export const traverse = async ({nodes, prev, cb}) => {
+  const waitings = nodes.reduce((acc, node) => {
+    acc[node] = getPromise()
+    return acc
+  }, {})
+
+  await Promise.all(nodes.map(async (name) => {
+    await Promise.all((prev.get(name) || []).map((p) => waitings[p].promise))
+    await cb(name)
+    waitings[name].resolve(true)
+  }))
+}

package/src/main/js/util.js

@@ -1,5 +1,3 @@
-import {$, fs} from 'zx-extra'
-
 export const tpl = (str, context) =>
   str?.replace(/\$\{\{\s*([.a-z0-9]+)\s*}}/gi, (matched, key) => get(context, key) ?? '')
 
@@ -30,73 +28,19 @@ export const set = (obj, path, value) => {
     return result
 }
 
-export const runHook = async (pkg, name) => {
-    const cmd = tpl(pkg.config[name], {...pkg, ...pkg.context})
-
-    if (cmd) {
-        log({pkg})(`run ${name} '${cmd}'`)
-        await $.o({cwd: pkg.absPath, quote: v => v, preferLocal: true})`${cmd}`
-    }
-}
-
-export const restJoin = (rest, context, def) => tpl(rest.filter(Boolean).join(' ') || def, context)
+export const msgJoin = (rest, context, def) => tpl(rest.filter(Boolean).join(' ') || def, context)
 
-export const createReporter = (file, logger = console) => {
-    const state = {
-        status: 'initial',
-        queue: [],
-        packages: [],
-        events: [],
-    }
+export const getPromise = () => {
+    let resolve, reject
+    const promise = new Promise((...args) => {
+        [resolve, reject] = args
+    })
 
     return {
-        setQueue(queue, packages) {
-            state.queue = queue
-            state.packages = queue.map(name => {
-                const {manifest: {version}, absPath, relPath} = packages[name]
-                return {
-                    status: 'initial',
-                    name,
-                    version,
-                    path: absPath,
-                    relPath
-                }
-            })
-        },
-        getState(key, pkgName) {
-            const _state = pkgName ? state.packages.find(({name}) => name === pkgName) : state
-            return get(_state, key)
-        },
-        setState(key, value, pkgName) {
-            const _state = pkgName ? state.packages.find(({name}) => name === pkgName) : state
-            set(_state, key, value)
-        },
-        setStatus(status, name) {
-            this.setState('status', status, name)
-
-            this.persistState()
-        },
-        getStatus (status, name) {
-            return this.getState('status', name)
-        },
-        log(ctx = {}) { return (...chunks) => {
-            const {pkg, scope = pkg?.name || '~', level = 'info'} = ctx
-            const msg = chunks.map(c => typeof c === 'string' ? tpl(c, ctx) : c)
-            const event = {msg, scope, date: Date.now(), level}
-            state.events.push(event)
-            logger[level](`[${scope}]`, ...msg)
-        }},
-        persistState() {
-            file && fs.outputJsonSync(file, state)
-        }
-    }
-}
-
-export const log = (ctx) => {
-    if (!$.r) {
-        $.r = createReporter()
+        resolve,
+        reject,
+        promise,
     }
-
-    return $.r.log(ctx)
 }
 
+export const keyByValue = (obj, value) => Object.keys(obj).find((key) => obj[key] === value)