presidium 2.1.0 → 3.0.1

package/S3Bucket.js

@@ -1309,11 +1309,12 @@ class S3Bucket {
       searchParams.set('versionId', options.VersionId)
     }
 
+    const encodedKey = encodeURIComponentRFC3986(key).replace(/%2F/g, '/')
     const response = await this._awsRequest1(
       'GET',
       searchParams.size > 0
-        ? `/${key}?${searchParams.toString()}`
-        : `/${key}`,
+        ? `/${encodedKey}?${searchParams.toString()}`
+        : `/${encodedKey}`,
       headers,
       ''
     )
@@ -1500,11 +1501,12 @@ class S3Bucket {
       searchParams.set('versionId', options.VersionId)
     }
 
+    const encodedKey = encodeURIComponentRFC3986(key).replace(/%2F/g, '/')
     const response = await this._awsRequest1(
       'GET',
       searchParams.size > 0
-        ? `/${key}?acl&${searchParams.toString()}`
-        : `/${key}?acl`,
+        ? `/${encodedKey}?acl&${searchParams.toString()}`
+        : `/${encodedKey}?acl`,
       headers,
       ''
     )
@@ -1750,11 +1752,12 @@ class S3Bucket {
       searchParams.set('versionId', options.VersionId)
     }
 
+    const encodedKey = encodeURIComponentRFC3986(key).replace(/%2F/g, '/')
     const response = await this._awsRequest1(
       'HEAD',
       searchParams.size > 0
-        ? `/${key}?${searchParams.toString()}`
-        : `/${key}`,
+        ? `/${encodedKey}?${searchParams.toString()}`
+        : `/${encodedKey}`,
       headers,
       ''
     )
@@ -1976,11 +1979,12 @@ class S3Bucket {
       searchParams.set('versionId', options.VersionId)
     }
 
+    const encodedKey = encodeURIComponentRFC3986(key).replace(/%2F/g, '/')
     const response = await this._awsRequest1(
       'DELETE',
       searchParams.size > 0
-        ? `/${key}?${searchParams.toString()}`
-        : `/${key}`,
+        ? `/${encodedKey}?${searchParams.toString()}`
+        : `/${encodedKey}`,
       headers,
       ''
     )

package/index.js

@@ -1,5 +1,4 @@
 module.exports = {
-  Archive: require('./Archive.js'),
   AwsCredentials: require('./AwsCredentials.js'),
   Docker: require('./Docker.js'),
   DynamoDBGlobalSecondaryIndex: require('./DynamoDBGlobalSecondaryIndex.js'),

package/internal/Archive.js

@@ -0,0 +1,113 @@
+require('rubico/global')
+const tarStream = require('tar-stream')
+const fs = require('fs/promises')
+const walk = require('./walk')
+const pipe = require('rubico/pipe')
+const tap = require('rubico/tap')
+const get = require('rubico/get')
+const reduce = require('rubico/reduce')
+const thunkify = require('rubico/thunkify')
+const pick = require('rubico/pick')
+
+/**
+ * @name Archive
+ */
+const Archive = {}
+
+/**
+ * @name Archive.tar
+ *
+ * @docs
+ * ```coffeescript [specscript]
+ * module tar 'https://github.com/mafintosh/tar-stream'
+ *
+ * Archive.tar(path string, options {
+ *   ignore: Array<string>,
+ *   base: Object<string>,
+ * }) -> pack tar.Pack
+ * ```
+ *
+ * Bundle multiple files and directories from a parent directory into a tarball.
+ *
+ * Arguments:
+ *   * `path` - the path to the parent directory which contains the files and directories to bundle.
+ *   * `options`
+ *     * `ignore` - filepaths, filenames, or patterns to ignore.
+ */
+Archive.tar = function tar(path, options) {
+  const pathWithSlash = path.endsWith('/') ? path : `${path}/`
+  const ignore = get(options, 'ignore')
+  const pack = tarStream.pack()
+
+  pipe(path, [
+    walk,
+    filter(path => {
+      for (const part of ignore) {
+        if (path.includes(part)) {
+          return false
+        }
+      }
+      return true
+    }),
+
+    reduce(async (pack, filepath) => {
+      pack.entry({
+        name: filepath.replace(pathWithSlash, ''),
+        ...pipe([
+          fs.stat,
+          pick(['size', 'mode', 'mtime', 'uid', 'gid']),
+        ])(filepath),
+      }, await fs.readFile(filepath))
+      return pack
+    }, pack),
+
+    tap(pack => {
+      const base = options.base ?? {}
+      for (const filename in base) {
+        pack.entry({ name: filename }, base[filename])
+      }
+      pack.finalize()
+    }),
+  ])
+
+  return pack
+}
+
+/**
+ * @name untar
+ *
+ * @docs
+ * ```coffeescript [specscript]
+ * module tar 'https://github.com/mafintosh/tar-stream'
+ * module presidium 'https://presidium.services/docs'
+ *
+ * Archive.untar(pack tar.Pack) ->
+ *   contentMap Promise<Map<(filepath string)=>(content presidium.Readable)> >
+ * ```
+ *
+ * Extract a tarball into a map of file paths to content streams.
+ *
+ * Arguments:
+ *   * `pack` - a tarball represented by the `tar.Pack` type. Returned by the [tar](#tar) method.
+ */
+Archive.untar = function untar(pack) {
+  const extractStream = tarStream.extract()
+  const extracted = new Map()
+  return new Promise((resolve, reject) => {
+    extractStream.on('entry', (header, stream, next) => {
+      stream.header = header
+      extracted.set(header.name, stream)
+      next()
+    })
+
+    extractStream.on('finish', () => {
+      resolve(extracted)
+    })
+
+    extractStream.on('error', reject)
+
+    pack.pipe(extractStream)
+  })
+}
+
+module.exports = Archive

package/internal/Archive.test.js

@@ -0,0 +1,71 @@
+const assert = require('assert')
+const stream = require('stream')
+const fs = require('fs/promises')
+const Test = require('thunk-test')
+const Archive = require('./Archive')
+const resolvePath = require('./resolvePath')
+const map = require('rubico/map')
+const reduce = require('rubico/reduce')
+const Readable = require('../Readable')
+
+const test = new Test('Archive', async function integration() {
+
+  {
+    const pack = await Archive.tar(__dirname, {
+      ignore: ['Dockerfile', 'node_modules', '.git', '.nyc_output'],
+    })
+    const extracted = await Archive.untar(pack)
+    assert(extracted.size > 0)
+    for (const [path, entry] of extracted) {
+      assert('header' in entry)
+      assert(!path.startsWith('/'))
+      assert.equal(typeof path, 'string')
+      assert.equal(typeof entry[Symbol.asyncIterator], 'function')
+      assert(entry.readable)
+    }
+  }
+
+  {
+    const base = {
+      Dockerfile: 'FROM node:15-alpine'
+    }
+    const ignore = ['fixtures']
+    const pack = await Archive.tar(__dirname, { ignore, base })
+    const extracted = await Archive.untar(pack)
+    const dir = await fs.readdir(__dirname)
+      .then(filter(n => !ignore.includes(n)))
+    const extractedKeys = [...extracted.keys()]
+    assert.equal(extracted.size, dir.length + 1) // extra Dockerfile
+    assert(extracted.has('Dockerfile'))
+    assert.equal(
+      await reduce(extracted.get('Dockerfile'), (a, b) => a + b, ''),
+      'FROM node:15-alpine'
+    )
+  }
+
+  {
+    const base = {
+      Dockerfile: 'FROM busybox:1.32',
+      '.aws/credentials': '[presidium]\naccessKeyId\nsecretAccessKey',
+    }
+    const pack = await Archive.tar(`${__dirname}/`, {
+      ignore: ['hashJSON.js'],
+      base,
+    })
+    const extracted = await Archive.untar(pack)
+    assert(extracted.size > 0)
+    assert(extracted.has('Dockerfile'))
+    assert(extracted.has('.aws/credentials'))
+    assert.equal(
+      await reduce(extracted.get('Dockerfile'), (a, b) => a + b, ''),
+      'FROM busybox:1.32'
+    )
+  }
+
+}).case()
+
+if (process.argv[1] == __filename) {
+  test()
+}
+
+module.exports = test

package/internal/GoogleChromeForTesting.js

@@ -0,0 +1,218 @@
+const https = require('https')
+const os = require('os')
+const fs = require('fs')
+const path = require('path')
+const { spawn } = require('child_process')
+const readline = require('readline')
+const extract = require('extract-zip')
+const HTTP = require('../HTTP')
+const XML = require('../XML')
+const Readable = require('../Readable')
+const walk = require('./walk')
+
+async function getChromeVersions() {
+  const http = new HTTP()
+  const response = await http.GET('https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json')
+  const data = await Readable.JSON(response)
+  return data
+}
+
+function updateConsoleLog(message) {
+  readline.cursorTo(process.stdout, 0, undefined);
+  readline.clearLine(process.stdout, 0);
+  process.stdout.write(message);
+}
+
+async function getChromeUrl() {
+  let platform = os.platform()
+  if (platform == 'darwin') {
+    platform = 'mac'
+  }
+  const arch = os.arch()
+
+  if (platform == 'mac') {
+    platform = `${platform}-${arch}`
+  } else {
+    platform = `${platform}${arch.slice(1)}`
+  }
+
+  let url
+  if (['stable', 'beta', 'dev', 'canary'].includes(this.chromeVersion)) {
+    const chromeVersions = await getChromeVersions()
+    const channel = `${this.chromeVersion[0].toUpperCase()}${this.chromeVersion.slice(1)}`
+    const chromeVersionNumber = chromeVersions.channels[channel].version
+    url = `https://storage.googleapis.com/chrome-for-testing-public/${chromeVersionNumber}/${platform}/chrome-${platform}.zip`
+  } else {
+    const chromeVersionNumber = this.chromeVersion
+    url = `https://storage.googleapis.com/chrome-for-testing-public/${chromeVersionNumber}/${platform}/chrome-${platform}.zip`
+  }
+
+  return url
+}
+
+async function installChrome() {
+  const url = await getChromeUrl.call(this)
+  let filepath = `${this.chromeDir}/${url.replace('https://storage.googleapis.com/chrome-for-testing-public/', '')}`
+  if (!filepath.startsWith('/')) {
+    filepath = path.join(process.cwd(), filepath)
+  }
+  let parentDir = `${filepath.split('/').slice(0, -1).join('/')}`
+  if (!parentDir.startsWith('/')) {
+    parentDir = path.join(process.cwd(), parentDir)
+  }
+  await fs.promises.mkdir(parentDir, { recursive: true })
+
+  const http = new HTTP()
+  const response = await http.GET(url)
+  const contentLength = response.headers['content-length']
+
+  await fs.promises.mkdir(parentDir, { recursive: true })
+  const fileStream = fs.createWriteStream(filepath)
+  // response.pipe(fileStream)
+
+  let downloadedLength = 0
+  response.on('data', chunk => {
+    downloadedLength += chunk.length
+    if (downloadedLength == contentLength) {
+      updateConsoleLog(`Downloading ${url} (${downloadedLength} / ${contentLength} bytes)\n`)
+    } else {
+      updateConsoleLog(`Downloading ${url} (${downloadedLength} / ${contentLength} bytes)`)
+    }
+
+    fileStream.write(chunk)
+  })
+
+  let resolve
+  const promise = new Promise(_resolve => {
+    resolve = _resolve
+  })
+  response.on('end', () => {
+    resolve()
+  })
+  await promise
+
+  await extract(filepath, { dir: parentDir })
+}
+
+async function getChromeFilepath() {
+  const url = await getChromeUrl.call(this)
+  const filepath = `${this.chromeDir}/${url.replace('https://storage.googleapis.com/chrome-for-testing-public/', '')}`
+  const parentDir = `${filepath.split('/').slice(0, -1).join('/')}`
+
+  const googleChromeForTestingFilepath = `${parentDir}`
+  try {
+    for await (const filepath of walk(parentDir)) {
+      console.log(filepath)
+      if (filepath.endsWith('Google Chrome for Testing')) {
+        return filepath
+      }
+    }
+  } catch (error) {
+    if (error.code == 'ENOENT') {
+      await installChrome.call(this)
+      const filepath = await getChromeFilepath.call(this)
+      return filepath
+    }
+    throw error
+  }
+
+  throw new Error('unable to find Google Chrome for Testing executable.')
+}
+
+/**
+ * @name GoogleChromeForTesting
+ *
+ * @docs
+ * ```coffeescript [specscript]
+ * new GoogleChromeForTesting(options {
+ *   chromeVersion: 'stable'|'beta'|'dev'|'canary'|string,
+ *   remoteDebuggingPort: number,
+ *   headless: boolean,
+ *   userDataDir: string,
+ *   useMockKeychain: boolean,
+ * }) -> GoogleChromeForTesting
+ * ```
+ *
+ * References:
+ *   * [Chrome for Testing availability](https://googlechromelabs.github.io/chrome-for-testing/)
+ */
+class GoogleChromeForTesting {
+  constructor(options = {}) {
+    this.chromeVersion = options.chromeVersion ?? 'stable'
+    this.chromeDir = options.chromeDir ?? 'google-chrome-for-testing'
+    this.remoteDebuggingPort = options.remoteDebuggingPort ?? 9222
+    this.headless = options.headless ?? false
+    this.userDataDir = options.userDataDir ?? './tmp/chrome'
+    this.useMockKeychain = options.useMockKeychain ?? false
+    this.devtoolsUrl = undefined
+  }
+
+  /**
+   * @name init
+   *
+   * @docs
+   * ```coffeescript [specscript]
+   * init() -> Promise<>
+   * ```
+   */
+  async init() {
+    const chromeFilepath = await getChromeFilepath.call(this)
+    console.log('spawn', chromeFilepath)
+
+    const cmd = spawn(chromeFilepath, [
+      `--remote-debugging-port=${this.remoteDebuggingPort}`,
+      `--user-data-dir=${this.userDataDir}`,
+      ...this.headless ? ['--headless'] : [],
+      ...this.useMockKeychain ? ['--use-mock-keychain'] : [],
+    ])
+    cmd.stdout.pipe(process.stdout)
+    cmd.stderr.pipe(process.stderr)
+
+    const devtoolsUrlPromiseWithResolvers = Promise.withResolvers()
+    cmd.stderr.on('data', chunk => {
+      const line = chunk.toString('utf8').trim()
+      if (line.includes('DevTools listening on')) {
+        const devtoolsUrl = line.replace('DevTools listening on ', '')
+        devtoolsUrlPromiseWithResolvers.resolve(devtoolsUrl)
+      }
+    })
+
+    const spawnPromiseWithResolvers = Promise.withResolvers()
+    cmd.on('spawn', () => {
+      spawnPromiseWithResolvers.resolve()
+    })
+
+    cmd.on('error', error => {
+      console.error(error)
+      process.exit(1)
+    })
+
+    process.on('SIGTERM', () => {
+      cmd.kill()
+    })
+
+    process.on('exit', () => {
+      cmd.kill()
+    })
+
+    this.cmd = cmd
+
+    await spawnPromiseWithResolvers.promise
+    this.devtoolsUrl = await devtoolsUrlPromiseWithResolvers.promise
+  }
+
+  /**
+   * @name close
+   *
+   * @docs
+   * ```coffeescript [specscript]
+   * close() -> undefined
+   * ```
+   */
+  close() {
+    this.cmd.kill('SIGKILL')
+  }
+
+}
+
+module.exports = GoogleChromeForTesting

package/internal/GoogleChromeForTesting.test.js

@@ -0,0 +1,50 @@
+const Test = require('thunk-test')
+const assert = require('assert')
+const fs = require('fs')
+const { exec } = require('child_process')
+const Readable = require('../Readable')
+const GoogleChromeForTesting = require('./GoogleChromeForTesting')
+
+const test = new Test('GoogleChromeForTesting', async function integration() {
+  await fs.promises.rm('google-chrome-for-testing', { recursive: true, force: true })
+
+  const cmd = await exec('ps aux | grep "Google Chrome for Testing" | awk \'{print $2}\' | xargs kill', {
+    stdio: 'inherit',
+  })
+
+  {
+    const googleChromeForTesting = new GoogleChromeForTesting({
+      userDataDir: `${__dirname}/tmp/chrome`,
+      useMockKeychain: true,
+    })
+    await googleChromeForTesting.init()
+
+    assert.equal(typeof googleChromeForTesting.devtoolsUrl, 'string')
+    assert(googleChromeForTesting.devtoolsUrl.startsWith('ws://'))
+
+    googleChromeForTesting.close()
+  }
+
+  {
+    const googleChromeForTesting = new GoogleChromeForTesting({
+      userDataDir: `${__dirname}/tmp/chrome`,
+      useMockKeychain: true,
+    })
+    await googleChromeForTesting.init()
+
+    assert.equal(typeof googleChromeForTesting.devtoolsUrl, 'string')
+    assert(googleChromeForTesting.devtoolsUrl.startsWith('ws://'))
+
+    googleChromeForTesting.close()
+  }
+
+  await fs.promises.rm('google-chrome-for-testing', { recursive: true, force: true })
+
+  console.log('Success')
+}).case()
+
+if (process.argv[1] == __filename) {
+  test()
+}
+
+module.exports = test

package/internal/parsePath.js

@@ -1,4 +1,4 @@
-const nodePath = require('path')
+const path = require('path')
 
 /**
  * @name parsePath
@@ -14,6 +14,6 @@ const nodePath = require('path')
  * }
  * ```
  */
-const parsePath = nodePath.parse
+const parsePath = path.parse
 
 module.exports = parsePath