presidium 4.8.1 → 4.9.0

package/Docker.js

@@ -16,6 +16,7 @@ const { spawn } = require('child_process')
 const Readable = require('./Readable')
 const HTTP = require('./HTTP')
 const querystring = require('querystring')
+const Archive = require('./internal/Archive')
 const split = require('./internal/split')
 const join = require('./internal/join')
 const isArray = require('./internal/isArray')
@@ -47,7 +48,6 @@ const handleDockerHTTPResponse = require('./internal/handleDockerHTTPResponse')
  *
  * System dependencies:
  *   * `docker`
- *   * `tar`
  *
  * Note: the Presidium Docker client connects to the Docker socket `/var/run/docker.sock`. For security purposes, please avoid exposing any code using the Presidium Docker client to the internet, see [How would an attacker gain access to the host machine from within a Docker container?](https://www.google.com/search?hl=en&q=how+would+an+attacker+gain+access+to+the+host+machine+from+within+a+docker+container&ia=web).
  */
@@ -408,54 +408,12 @@ class Docker {
    *   * [Dockerfile](https://docs.docker.com/engine/reference/builder/)
    */
   async buildImage(path, options = {}) {
-    path = path.endsWith('/') ? path : `${path}/`
-
-    const archiveDir = `presidium-Docker-buildImage-tmp-${uniqid()}`
-
-    await fs.promises.mkdir(archiveDir, { recursive: true })
-
-    const archiveFilename = `${archiveDir}/archive.tar`
-
-    const cmdArgs = []
-
-    cmdArgs.push(`--exclude=${archiveDir}`)
-    if (options.ignore) {
-      for (const pattern of options.ignore) {
-        cmdArgs.push(`--exclude=${pattern}`)
-      }
-    }
-
-    cmdArgs.push('-cf', archiveFilename, '-C', path, '.')
-
-    const cmd = spawn('tar', cmdArgs, { stdio: 'inherit' })
-
-    await new Promise(resolve => {
-      cmd.on('close', resolve)
+    const pack = Archive.tar(path, {
+      base: options.archive,
+      ignore: options.ignore ?? ['node_modules', '.git', '.nyc_output'],
     })
 
-    if (options.archive) {
-      for (const filename in options.archive) {
-        await fs.promises.writeFile(
-          `${archiveDir}/${filename}`, options.archive[filename]
-        )
-
-        const cmdArgs = [
-          '-rf',
-          archiveFilename,
-          `--directory=${archiveDir}`,
-        ]
-
-        cmdArgs.push(filename)
-
-        const cmd = spawn('tar', cmdArgs, { stdio: 'inherit' })
-
-        await new Promise(resolve => {
-          cmd.on('close', resolve)
-        })
-      }
-    }
-
-    const archive = fs.createReadStream(archiveFilename).pipe(zlib.createGzip())
+    const compressed = pack.pipe(zlib.createGzip())
 
     const response = await this.http.post(`/build?${querystring.stringify({
       dockerfile: options.archiveDockerfile ?? 'Dockerfile',
@@ -463,18 +421,13 @@ class Docker {
       forcerm: true,
       platform: options.platform ?? '',
     })}`, {
-      body: archive,
+      body: compressed,
       headers: {
         'Content-Type': 'application/x-tar',
       },
     })
 
     const dataStream = await handleDockerHTTPResponse(response, { stream: true })
-
-    dataStream.on('end', async () => {
-      await fs.promises.rm(archiveDir, { recursive: true })
-    })
-
     return dataStream
   }
 

package/internal/Archive.js

@@ -0,0 +1,118 @@
+require('rubico/global')
+const tarStream = require('tar-stream')
+const fs = require('fs')
+const walk = require('./walk')
+
+/**
+ * @name Archive
+ */
+const Archive = {
+
+  /**
+   * @name tar
+   *
+   * @docs
+   * ```coffeescript [specscript]
+   * module tar 'https://github.com/mafintosh/tar-stream'
+   *
+   * Archive.tar(path string, options {
+   *   ignore: Array<string>,
+   *   base: Object<string>,
+   * }) -> pack tar.Pack
+   * ```
+   *
+   * Bundle multiple files and directories from a parent directory into a tarball.
+   *
+   * Arguments:
+   *   * `path` - the path to the parent directory which contains the files and directories to bundle.
+   *   * `options`
+   *     * `ignore` - filepaths, filenames, or patterns to ignore.
+   */
+  tar(path, options) {
+    path = path.endsWith('/') ? path : `${path}/`
+    const ignore = get(options, 'ignore')
+    const pack = tarStream.pack()
+
+    pipe(
+      walk(path),
+
+      filter(path => {
+        for (const part of ignore) {
+          if (path.includes(part)) {
+            return false
+          }
+        }
+        return true
+      }),
+
+      reduce(async (pack, filepath) => {
+        const stats = await pipe(filepath, [
+          fs.promises.stat,
+          pick(['size', 'mode', 'mtime', 'uid', 'gid']),
+        ])
+
+        const entry = pack.entry({ ...stats, name: filepath.replace(path, '') })
+
+        const fileStream = fs.createReadStream(filepath)
+
+        fileStream.pipe(entry)
+
+        await new Promise(resolve => {
+          fileStream.on('end', resolve)
+        })
+
+        return pack
+      }, pack),
+
+      tap(pack => {
+        const base = options.base ?? {}
+        for (const filename in base) {
+          pack.entry({ name: filename }, base[filename])
+        }
+        pack.finalize()
+      }),
+    )
+
+    return pack
+  },
+
+  /**
+   * @name untar
+   *
+   * @docs
+   * ```coffeescript [specscript]
+   * module tar 'https://github.com/mafintosh/tar-stream'
+   * module Readable 'https://presidium.services/Readable'
+   *
+   * Archive.untar(pack tar.Pack) ->
+   *   contentMap Promise<Map<(filepath string)=>(content Readable)> >
+   * ```
+   *
+   * Extract a tarball into a map of file paths to content streams.
+   *
+   * Arguments:
+   *   * `pack` - a tarball represented by the `tar.Pack` type. Returned by the [tar](#tar) method.
+   */
+  untar(pack) {
+    const extractStream = tarStream.extract()
+    const extracted = new Map()
+    return new Promise((resolve, reject) => {
+      extractStream.on('entry', (header, stream, next) => {
+        stream.header = header
+        extracted.set(header.name, stream)
+        next()
+      })
+
+      extractStream.on('finish', () => {
+        resolve(extracted)
+      })
+
+      extractStream.on('error', reject)
+
+      pack.pipe(extractStream)
+    })
+  },
+
+}
+
+module.exports = Archive

package/internal/Archive.test.js

@@ -0,0 +1,78 @@
+const assert = require('assert')
+const stream = require('stream')
+const fs = require('fs/promises')
+const Test = require('thunk-test')
+const Archive = require('./Archive')
+const resolvePath = require('./resolvePath')
+const map = require('rubico/map')
+const reduce = require('rubico/reduce')
+const Readable = require('../Readable')
+
+const test = new Test('Archive', async function integration() {
+
+  {
+    const pack = await Archive.tar(__dirname, {
+      ignore: ['Dockerfile', 'node_modules', '.git', '.nyc_output'],
+    })
+    const extracted = await Archive.untar(pack)
+    assert(extracted.size > 0)
+    for (const [path, entry] of extracted) {
+      assert('header' in entry)
+      assert(!path.startsWith('/'))
+      assert.equal(typeof path, 'string')
+      assert.equal(typeof entry[Symbol.asyncIterator], 'function')
+      assert(entry.readable)
+    }
+  }
+
+  {
+    const base = {
+      Dockerfile: 'FROM node:15-alpine'
+    }
+    const ignore = ['fixtures', 'tmp', 'google-chrome-for-testing']
+    const pack = await Archive.tar(__dirname, { ignore, base })
+    const extracted = await Archive.untar(pack)
+    const dir = await fs.readdir(__dirname)
+      .then(filter(filepath => {
+        for (const part of ignore) {
+          if (filepath.includes(part)) {
+            return false
+          }
+        }
+        return true
+      }))
+    const extractedKeys = [...extracted.keys()]
+    assert.equal(extracted.size, dir.length + 1) // extra Dockerfile
+    assert(extracted.has('Dockerfile'))
+    assert.equal(
+      await reduce(extracted.get('Dockerfile'), (a, b) => a + b, ''),
+      'FROM node:15-alpine'
+    )
+  }
+
+  {
+    const base = {
+      Dockerfile: 'FROM busybox:1.32',
+      '.aws/credentials': '[presidium]\naccessKeyId\nsecretAccessKey',
+    }
+    const pack = await Archive.tar(`${__dirname}/`, {
+      ignore: ['hashJSON.js'],
+      base,
+    })
+    const extracted = await Archive.untar(pack)
+    assert(extracted.size > 0)
+    assert(extracted.has('Dockerfile'))
+    assert(extracted.has('.aws/credentials'))
+    assert.equal(
+      await reduce(extracted.get('Dockerfile'), (a, b) => a + b, ''),
+      'FROM busybox:1.32'
+    )
+  }
+
+}).case()
+
+if (process.argv[1] == __filename) {
+  test()
+}
+
+module.exports = test

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "presidium",
-  "version": "4.8.1",
+  "version": "4.9.0",
   "description": "The Presidium library.",
   "author": "Richard Tong",
   "license": "CFOSS",
@@ -68,9 +68,10 @@
     "adm-zip": "^0.6.1",
     "html-entities": "^2.6.0",
     "presidium-db": ">=4.6.2",
-    "presidium-db-client-server": ">=0.6.3",
+    "presidium-db-client-server": ">=0.6.6",
     "presidium-websocket": ">=4.2.2",
     "rubico": "^2.21.0",
+    "tar-stream": "^3.2.2",
     "uniqid": "^5.4.0"
   },
   "scripts": {