step-node-agent 3.16.3 → 3.18.0

package/node_modules/npm/node_modules/tar/lib/unpack.js

@@ -1,5 +1,11 @@
 'use strict'
 
+// the PEND/UNPEND stuff tracks whether we're ready to emit end/close yet.
+// but the path reservations are required to avoid race conditions where
+// parallelized unpack ops may mess with one another, due to dependencies
+// (like a Link depending on its target) or destructive operations (like
+// clobbering an fs object to create one of a different type.)
+
 const assert = require('assert')
 const EE = require('events').EventEmitter
 const Parser = require('./parse.js')
@@ -9,9 +15,15 @@ const path = require('path')
 const mkdir = require('./mkdir.js')
 const mkdirSync = mkdir.sync
 const wc = require('./winchars.js')
+const stripAbsolutePath = require('./strip-absolute-path.js')
+const pathReservations = require('./path-reservations.js')
+const normPath = require('./normalize-windows-path.js')
+const stripSlash = require('./strip-trailing-slashes.js')
 
 const ONENTRY = Symbol('onEntry')
 const CHECKFS = Symbol('checkFs')
+const CHECKFS2 = Symbol('checkFs2')
+const PRUNECACHE = Symbol('pruneCache')
 const ISREUSABLE = Symbol('isReusable')
 const MAKEFS = Symbol('makeFs')
 const FILE = Symbol('file')
@@ -33,7 +45,10 @@ const SKIP = Symbol('skip')
 const DOCHOWN = Symbol('doChown')
 const UID = Symbol('uid')
 const GID = Symbol('gid')
+const CHECKED_CWD = Symbol('checkedCwd')
 const crypto = require('crypto')
+const platform = process.env.TESTING_TAR_FAKE_PLATFORM || process.platform
+const isWindows = platform === 'win32'
 
 // Unlinks on Windows are not atomic.
 //
@@ -52,7 +67,7 @@ const crypto = require('crypto')
 // See: https://github.com/npm/node-tar/issues/183
 /* istanbul ignore next */
 const unlinkFile = (path, cb) => {
-  if (process.platform !== 'win32')
+  if (!isWindows)
     return fs.unlink(path, cb)
 
   const name = path + '.DELETE.' + crypto.randomBytes(16).toString('hex')
@@ -65,7 +80,7 @@ const unlinkFile = (path, cb) => {
 
 /* istanbul ignore next */
 const unlinkFileSync = path => {
-  if (process.platform !== 'win32')
+  if (!isWindows)
     return fs.unlinkSync(path)
 
   const name = path + '.DELETE.' + crypto.randomBytes(16).toString('hex')
@@ -79,6 +94,33 @@ const uint32 = (a, b, c) =>
   : b === b >>> 0 ? b
   : c
 
+// clear the cache if it's a case-insensitive unicode-squashing match.
+// we can't know if the current file system is case-sensitive or supports
+// unicode fully, so we check for similarity on the maximally compatible
+// representation.  Err on the side of pruning, since all it's doing is
+// preventing lstats, and it's not the end of the world if we get a false
+// positive.
+// Note that on windows, we always drop the entire cache whenever a
+// symbolic link is encountered, because 8.3 filenames are impossible
+// to reason about, and collisions are hazards rather than just failures.
+const cacheKeyNormalize = path => stripSlash(normPath(path))
+  .normalize('NFKD')
+  .toLowerCase()
+
+const pruneCache = (cache, abs) => {
+  abs = cacheKeyNormalize(abs)
+  for (const path of cache.keys()) {
+    const pnorm = cacheKeyNormalize(path)
+    if (pnorm === abs || pnorm.indexOf(abs + '/') === 0)
+      cache.delete(path)
+  }
+}
+
+const dropCache = cache => {
+  for (const key of cache.keys())
+    cache.delete(key)
+}
+
 class Unpack extends Parser {
   constructor (opt) {
     if (!opt)
@@ -91,6 +133,10 @@ class Unpack extends Parser {
 
     super(opt)
 
+    this[CHECKED_CWD] = false
+
+    this.reservations = pathReservations()
+
     this.transform = typeof opt.transform === 'function' ? opt.transform : null
 
     this.writable = true
@@ -133,7 +179,7 @@ class Unpack extends Parser {
     this.forceChown = opt.forceChown === true
 
     // turn ><?| in filenames into 0xf000-higher encoded forms
-    this.win32 = !!opt.win32 || process.platform === 'win32'
+    this.win32 = !!opt.win32 || isWindows
 
     // do not unpack over files that are newer than what's in the archive
     this.newer = !!opt.newer
@@ -153,7 +199,7 @@ class Unpack extends Parser {
     // links, and removes symlink directories rather than erroring
     this.unlink = !!opt.unlink
 
-    this.cwd = path.resolve(opt.cwd || process.cwd())
+    this.cwd = normPath(path.resolve(opt.cwd || process.cwd()))
     this.strip = +opt.strip || 0
     this.processUmask = process.umask()
     this.umask = typeof opt.umask === 'number' ? opt.umask : this.processUmask
@@ -174,46 +220,72 @@ class Unpack extends Parser {
 
   [CHECKPATH] (entry) {
     if (this.strip) {
-      const parts = entry.path.split(/\/|\\/)
+      const parts = normPath(entry.path).split('/')
       if (parts.length < this.strip)
         return false
       entry.path = parts.slice(this.strip).join('/')
 
       if (entry.type === 'Link') {
-        const linkparts = entry.linkpath.split(/\/|\\/)
+        const linkparts = normPath(entry.linkpath).split('/')
         if (linkparts.length >= this.strip)
           entry.linkpath = linkparts.slice(this.strip).join('/')
+        else
+          return false
       }
     }
 
     if (!this.preservePaths) {
-      const p = entry.path
-      if (p.match(/(^|\/|\\)\.\.(\\|\/|$)/)) {
-        this.warn('path contains \'..\'', p)
+      const p = normPath(entry.path)
+      const parts = p.split('/')
+      if (parts.includes('..') || isWindows && /^[a-z]:\.\.$/i.test(parts[0])) {
+        this.warn(`path contains '..'`, p)
         return false
       }
 
-      // absolutes on posix are also absolutes on win32
-      // so we only need to test this one to get both
-      if (path.win32.isAbsolute(p)) {
-        const parsed = path.win32.parse(p)
-        this.warn('stripping ' + parsed.root + ' from absolute path', p)
-        entry.path = p.substr(parsed.root.length)
+      // strip off the root
+      const s = stripAbsolutePath(p)
+      if (s[0]) {
+        entry.path = s[1]
+        this.warn(`stripping ${s[0]} from absolute path`, p)
       }
     }
 
+    if (path.isAbsolute(entry.path))
+      entry.absolute = normPath(path.resolve(entry.path))
+    else
+      entry.absolute = normPath(path.resolve(this.cwd, entry.path))
+
+    // if we somehow ended up with a path that escapes the cwd, and we are
+    // not in preservePaths mode, then something is fishy!  This should have
+    // been prevented above, so ignore this for coverage.
+    /* istanbul ignore if - defense in depth */
+    if (!this.preservePaths &&
+        entry.absolute.indexOf(this.cwd + '/') !== 0 &&
+        entry.absolute !== this.cwd) {
+      this.warn('TAR_ENTRY_ERROR', 'path escaped extraction target', {
+        entry,
+        path: normPath(entry.path),
+        resolvedPath: entry.absolute,
+        cwd: this.cwd,
+      })
+      return false
+    }
+
+    // an archive can set properties on the extraction directory, but it
+    // may not replace the cwd with a different kind of thing entirely.
+    if (entry.absolute === this.cwd &&
+        entry.type !== 'Directory' &&
+        entry.type !== 'GNUDumpDir')
+      return false
+
     // only encode : chars that aren't drive letter indicators
     if (this.win32) {
-      const parsed = path.win32.parse(entry.path)
-      entry.path = parsed.root === '' ? wc.encode(entry.path)
-        : parsed.root + wc.encode(entry.path.substr(parsed.root.length))
+      const { root: aRoot } = path.win32.parse(entry.absolute)
+      entry.absolute = aRoot + wc.encode(entry.absolute.substr(aRoot.length))
+      const { root: pRoot } = path.win32.parse(entry.path)
+      entry.path = pRoot + wc.encode(entry.path.substr(pRoot.length))
     }
 
-    if (path.isAbsolute(entry.path))
-      entry.absolute = entry.path
-    else
-      entry.absolute = path.resolve(this.cwd, entry.path)
-
     return true
   }
 
@@ -257,7 +329,7 @@ class Unpack extends Parser {
   }
 
   [MKDIR] (dir, mode, cb) {
-    mkdir(dir, {
+    mkdir(normPath(dir), {
       uid: this.uid,
       gid: this.gid,
       processUid: this.processUid,
@@ -291,21 +363,44 @@ class Unpack extends Parser {
     return uint32(this.gid, entry.gid, this.processGid)
   }
 
-  [FILE] (entry) {
+  [FILE] (entry, fullyDone) {
     const mode = entry.mode & 0o7777 || this.fmode
     const stream = new fsm.WriteStream(entry.absolute, {
       mode: mode,
       autoClose: false
     })
-    stream.on('error', er => this[ONERROR](er, entry))
+    stream.on('error', er => {
+      if (stream.fd)
+        fs.close(stream.fd, () => {})
+
+      // flush all the data out so that we aren't left hanging
+      // if the error wasn't actually fatal.  otherwise the parse
+      // is blocked, and we never proceed.
+      /* istanbul ignore next */
+      stream.write = () => true
+      this[ONERROR](er, entry)
+      fullyDone()
+    })
 
     let actions = 1
     const done = er => {
-      if (er)
-        return this[ONERROR](er, entry)
+      if (er) {
+        /* istanbul ignore else - we should always have a fd by now */
+        if (stream.fd)
+          fs.close(stream.fd, () => {})
 
-      if (--actions === 0)
-        fs.close(stream.fd, _ => this[UNPEND]())
+        this[ONERROR](er, entry)
+        fullyDone()
+        return
+      }
+
+      if (--actions === 0) {
+        fs.close(stream.fd, er => {
+          fullyDone()
+          /* istanbul ignore next */
+          er ? this[ONERROR](er, entry) : this[UNPEND]()
+        })
+      }
     }
 
     stream.on('finish', _ => {
@@ -344,15 +439,18 @@ class Unpack extends Parser {
     tx.pipe(stream)
   }
 
-  [DIRECTORY] (entry) {
+  [DIRECTORY] (entry, fullyDone) {
     const mode = entry.mode & 0o7777 || this.dmode
     this[MKDIR](entry.absolute, mode, er => {
-      if (er)
+      if (er) {
+        fullyDone()
         return this[ONERROR](er, entry)
+      }
 
       let actions = 1
       const done = _ => {
         if (--actions === 0) {
+          fullyDone()
           this[UNPEND]()
           entry.resume()
         }
@@ -377,12 +475,13 @@ class Unpack extends Parser {
     entry.resume()
   }
 
-  [SYMLINK] (entry) {
-    this[LINK](entry, entry.linkpath, 'symlink')
+  [SYMLINK] (entry, done) {
+    this[LINK](entry, entry.linkpath, 'symlink', done)
   }
 
-  [HARDLINK] (entry) {
-    this[LINK](entry, path.resolve(this.cwd, entry.linkpath), 'link')
+  [HARDLINK] (entry, done) {
+    const linkpath = normPath(path.resolve(this.cwd, entry.linkpath))
+    this[LINK](entry, linkpath, 'link', done)
   }
 
   [PEND] () {
@@ -407,35 +506,122 @@ class Unpack extends Parser {
       !this.unlink &&
       st.isFile() &&
       st.nlink <= 1 &&
-      process.platform !== 'win32'
+      !isWindows
   }
 
   // check if a thing is there, and if so, try to clobber it
   [CHECKFS] (entry) {
     this[PEND]()
-    this[MKDIR](path.dirname(entry.absolute), this.dmode, er => {
-      if (er)
-        return this[ONERROR](er, entry)
-      fs.lstat(entry.absolute, (er, st) => {
-        if (st && (this.keep || this.newer && st.mtime > entry.mtime))
+    const paths = [entry.path]
+    if (entry.linkpath)
+      paths.push(entry.linkpath)
+    this.reservations.reserve(paths, done => this[CHECKFS2](entry, done))
+  }
+
+  [PRUNECACHE] (entry) {
+    // if we are not creating a directory, and the path is in the dirCache,
+    // then that means we are about to delete the directory we created
+    // previously, and it is no longer going to be a directory, and neither
+    // is any of its children.
+    // If a symbolic link is encountered, all bets are off.  There is no
+    // reasonable way to sanitize the cache in such a way we will be able to
+    // avoid having filesystem collisions.  If this happens with a non-symlink
+    // entry, it'll just fail to unpack, but a symlink to a directory, using an
+    // 8.3 shortname or certain unicode attacks, can evade detection and lead
+    // to arbitrary writes to anywhere on the system.
+    if (entry.type === 'SymbolicLink')
+      dropCache(this.dirCache)
+    else if (entry.type !== 'Directory')
+      pruneCache(this.dirCache, entry.absolute)
+  }
+
+  [CHECKFS2] (entry, fullyDone) {
+    this[PRUNECACHE](entry)
+
+    const done = er => {
+      this[PRUNECACHE](entry)
+      fullyDone(er)
+    }
+
+    const checkCwd = () => {
+      this[MKDIR](this.cwd, this.dmode, er => {
+        if (er) {
+          this[ONERROR](er, entry)
+          done()
+          return
+        }
+        this[CHECKED_CWD] = true
+        start()
+      })
+    }
+
+    const start = () => {
+      if (entry.absolute !== this.cwd) {
+        const parent = normPath(path.dirname(entry.absolute))
+        if (parent !== this.cwd) {
+          return this[MKDIR](parent, this.dmode, er => {
+            if (er) {
+              this[ONERROR](er, entry)
+              done()
+              return
+            }
+            afterMakeParent()
+          })
+        }
+      }
+      afterMakeParent()
+    }
+
+    const afterMakeParent = () => {
+      fs.lstat(entry.absolute, (lstatEr, st) => {
+        if (st && (this.keep || this.newer && st.mtime > entry.mtime)) {
           this[SKIP](entry)
-        else if (er || this[ISREUSABLE](entry, st))
-          this[MAKEFS](null, entry)
-        else if (st.isDirectory()) {
+          done()
+          return
+        }
+        if (lstatEr || this[ISREUSABLE](entry, st))
+          return this[MAKEFS](null, entry, done)
+
+        if (st.isDirectory()) {
           if (entry.type === 'Directory') {
-            if (!entry.mode || (st.mode & 0o7777) === entry.mode)
-              this[MAKEFS](null, entry)
-            else
-              fs.chmod(entry.absolute, entry.mode, er => this[MAKEFS](er, entry))
-          } else
-            fs.rmdir(entry.absolute, er => this[MAKEFS](er, entry))
-        } else
-          unlinkFile(entry.absolute, er => this[MAKEFS](er, entry))
+            const needChmod = !this.noChmod &&
+              entry.mode &&
+              (st.mode & 0o7777) !== entry.mode
+            const afterChmod = er => this[MAKEFS](er, entry, done)
+            if (!needChmod)
+              return afterChmod()
+            return fs.chmod(entry.absolute, entry.mode, afterChmod)
+          }
+          // Not a dir entry, have to remove it.
+          // NB: the only way to end up with an entry that is the cwd
+          // itself, in such a way that == does not detect, is a
+          // tricky windows absolute path with UNC or 8.3 parts (and
+          // preservePaths:true, or else it will have been stripped).
+          // In that case, the user has opted out of path protections
+          // explicitly, so if they blow away the cwd, c'est la vie.
+          if (entry.absolute !== this.cwd) {
+            return fs.rmdir(entry.absolute, er =>
+              this[MAKEFS](er, entry, done))
+          }
+        }
+
+        // not a dir, and not reusable
+        // don't remove if the cwd, we want that error
+        if (entry.absolute === this.cwd)
+          return this[MAKEFS](null, entry, done)
+
+        unlinkFile(entry.absolute, er =>
+          this[MAKEFS](er, entry, done))
       })
-    })
+    }
+
+    if (this[CHECKED_CWD])
+      start()
+    else
+      checkCwd()
   }
 
-  [MAKEFS] (er, entry) {
+  [MAKEFS] (er, entry, done) {
     if (er)
       return this[ONERROR](er, entry)
 
@@ -443,73 +629,107 @@ class Unpack extends Parser {
       case 'File':
       case 'OldFile':
       case 'ContiguousFile':
-        return this[FILE](entry)
+        return this[FILE](entry, done)
 
       case 'Link':
-        return this[HARDLINK](entry)
+        return this[HARDLINK](entry, done)
 
       case 'SymbolicLink':
-        return this[SYMLINK](entry)
+        return this[SYMLINK](entry, done)
 
       case 'Directory':
       case 'GNUDumpDir':
-        return this[DIRECTORY](entry)
+        return this[DIRECTORY](entry, done)
     }
   }
 
-  [LINK] (entry, linkpath, link) {
-    // XXX: get the type ('file' or 'dir') for windows
+  [LINK] (entry, linkpath, link, done) {
+    // XXX: get the type ('symlink' or 'junction') for windows
     fs[link](linkpath, entry.absolute, er => {
       if (er)
         return this[ONERROR](er, entry)
+      done()
       this[UNPEND]()
       entry.resume()
     })
   }
 }
 
+const callSync = fn => {
+  try {
+    return [null, fn()]
+  } catch (er) {
+    return [er, null]
+  }
+}
 class UnpackSync extends Unpack {
-  constructor (opt) {
-    super(opt)
+  [MAKEFS] (er, entry) {
+    return super[MAKEFS](er, entry, /* istanbul ignore next */ () => {})
   }
 
   [CHECKFS] (entry) {
-    const er = this[MKDIR](path.dirname(entry.absolute), this.dmode)
-    if (er)
-      return this[ONERROR](er, entry)
-    try {
-      const st = fs.lstatSync(entry.absolute)
-      if (this.keep || this.newer && st.mtime > entry.mtime)
-        return this[SKIP](entry)
-      else if (this[ISREUSABLE](entry, st))
-        return this[MAKEFS](null, entry)
-      else {
-        try {
-          if (st.isDirectory()) {
-            if (entry.type === 'Directory') {
-              if (entry.mode && (st.mode & 0o7777) !== entry.mode)
-                fs.chmodSync(entry.absolute, entry.mode)
-            } else
-              fs.rmdirSync(entry.absolute)
-          } else
-            unlinkFileSync(entry.absolute)
-          return this[MAKEFS](null, entry)
-        } catch (er) {
-          return this[ONERROR](er, entry)
-        }
+    this[PRUNECACHE](entry)
+
+    if (!this[CHECKED_CWD]) {
+      const er = this[MKDIR](this.cwd, this.dmode)
+      if (er)
+        return this[ONERROR](er, entry)
+      this[CHECKED_CWD] = true
+    }
+
+    // don't bother to make the parent if the current entry is the cwd,
+    // we've already checked it.
+    if (entry.absolute !== this.cwd) {
+      const parent = normPath(path.dirname(entry.absolute))
+      if (parent !== this.cwd) {
+        const mkParent = this[MKDIR](parent, this.dmode)
+        if (mkParent)
+          return this[ONERROR](mkParent, entry)
       }
-    } catch (er) {
+    }
+
+    const [lstatEr, st] = callSync(() => fs.lstatSync(entry.absolute))
+    if (st && (this.keep || this.newer && st.mtime > entry.mtime))
+      return this[SKIP](entry)
+
+    if (lstatEr || this[ISREUSABLE](entry, st))
       return this[MAKEFS](null, entry)
+
+    if (st.isDirectory()) {
+      if (entry.type === 'Directory') {
+        const needChmod = !this.noChmod &&
+          entry.mode &&
+          (st.mode & 0o7777) !== entry.mode
+        const [er] = needChmod ? callSync(() => {
+          fs.chmodSync(entry.absolute, entry.mode)
+        }) : []
+        return this[MAKEFS](er, entry)
+      }
+      // not a dir entry, have to remove it
+      const [er] = callSync(() => fs.rmdirSync(entry.absolute))
+      this[MAKEFS](er, entry)
     }
+
+    // not a dir, and not reusable.
+    // don't remove if it's the cwd, since we want that error.
+    const [er] = entry.absolute === this.cwd ? []
+      : callSync(() => unlinkFileSync(entry.absolute))
+    this[MAKEFS](er, entry)
   }
 
-  [FILE] (entry) {
+  [FILE] (entry, done) {
     const mode = entry.mode & 0o7777 || this.fmode
 
     const oner = er => {
-      try { fs.closeSync(fd) } catch (_) {}
-      if (er)
-        this[ONERROR](er, entry)
+      let closeError
+      try {
+        fs.closeSync(fd)
+      } catch (e) {
+        closeError = e
+      }
+      if (er || closeError)
+        this[ONERROR](er || closeError, entry)
+      done()
     }
 
     let stream
@@ -570,11 +790,14 @@ class UnpackSync extends Unpack {
     })
   }
 
-  [DIRECTORY] (entry) {
+  [DIRECTORY] (entry, done) {
     const mode = entry.mode & 0o7777 || this.dmode
     const er = this[MKDIR](entry.absolute, mode)
-    if (er)
-      return this[ONERROR](er, entry)
+    if (er) {
+      this[ONERROR](er, entry)
+      done()
+      return
+    }
     if (entry.mtime && !this.noMtime) {
       try {
         fs.utimesSync(entry.absolute, entry.atime || new Date(), entry.mtime)
@@ -585,12 +808,13 @@ class UnpackSync extends Unpack {
         fs.chownSync(entry.absolute, this[UID](entry), this[GID](entry))
       } catch (er) {}
     }
+    done()
     entry.resume()
   }
 
   [MKDIR] (dir, mode) {
     try {
-      return mkdir.sync(dir, {
+      return mkdir.sync(normPath(dir), {
         uid: this.uid,
         gid: this.gid,
         processUid: this.processUid,
@@ -607,9 +831,10 @@ class UnpackSync extends Unpack {
     }
   }
 
-  [LINK] (entry, linkpath, link) {
+  [LINK] (entry, linkpath, link, done) {
     try {
       fs[link + 'Sync'](linkpath, entry.absolute)
+      done()
       entry.resume()
     } catch (er) {
       return this[ONERROR](er, entry)