reviewable-enterprise-tools 1.1.0 → 1.3.1

package/extract_data.js

@@ -2,13 +2,16 @@
 
 import _ from 'lodash';
 import * as fs from 'fs';
+import * as path from 'path';
 import * as zlib from 'zlib';
 import commandLineArgs from 'command-line-args';
 import getUsage from 'command-line-usage';
 import {forEachLimit, forEachOfLimit, forEachOf} from 'async';
 import nodefireModule from 'nodefire';
 import {PromiseWritable} from 'promise-writable';
+import {default as download} from 'download';
 import Pace from 'pace';
+import {uploadedFilesUrl, PLACEHOLDER_URL} from './lib/derivedInfo.js';
 
 const NodeFire = nodefireModule.default;
 
@@ -20,6 +23,10 @@ const commandLineOptions = [
       'user id mappings. (Optional, defaults to identity mapping.)'},
   {name: 'output', alias: 'o', typeLabel: '{underline data.ndjson}',
     description: 'Output ndJSON file for extracted data.'},
+  {name: 'download', alias: 'd', typeLabel: '{underline file/download/dir}',
+    description: 'Output directory for downloaded attachments'},
+  {name: 'logging', alias: 'l', type: Boolean,
+    description: 'Turn on low-level Firebase logging for debugging purposes'},
   {name: 'help', alias: 'h', type: Boolean,
     description: 'Display these usage instructions.'}
 ];
@@ -45,6 +52,17 @@ for (const property of ['repos', 'output']) {
   }
 }
 
+let uploadedFilesUrlRegex;
+if (uploadedFilesUrl) {
+  uploadedFilesUrlRegex = new RegExp(`(${_.escapeRegExp(uploadedFilesUrl)})([^)]*)`, 'g');
+} else {
+  console.warn(
+    'WARNING: no REVIEWABLE_UPLOADS_PROVIDER or REVIEWABLE_UPLOADED_FILES_URL specified, ' +
+    'so not rewriting uploaded image URLs in comments.');
+}
+
+if (args.logging) NodeFire.enableFirebaseLogging(true);
+
 const identityUserMap = !args.users;
 const userMap = args.users ? JSON.parse(fs.readFileSync(args.users)) : {};
 const repoNames =
@@ -55,14 +73,19 @@ const orgNames = _(repoNames).map(name => name.replace(/\/.*/, '')).uniq().value
 const out = new PromiseWritable(fs.createWriteStream(args.output));
 out.stream.setMaxListeners(Infinity);
 
-const pace = Pace(1 + 2 + orgNames.length + 2 * repoNames.length + _.size(userMap));
+const pace = args.logging ?
+  {op() {}, total: 0} :
+  Pace(1 + 2 + orgNames.length + 2 * repoNames.length + _.size(userMap));
 
 let reviewKeys = [];
 const reversePullRequests = {};
 let ghostedUsers = [];
 const missingReviewKeys = [];
+const brokenFiles = [];
+const downloadedFiles = new Set();
 
 async function extract() {
+  log('Connecting to Firebase');
   await import('./lib/loadFirebase.js');
   await extractSystem();
   await extractOrganizations();
@@ -76,8 +99,15 @@ async function extract() {
   await extractUsers();
   await out.end();
   pace.op();
+  console.log(
+    `Extracted ${orgNames.length} organizations, ${repoNames.length} repositories, ` +
+    `${reviewKeys.length} reviews, ${args.download ? '' : 'and '}${_.size(userMap)} users` + (
+      args.download ? `, and ${downloadedFiles.size - brokenFiles.length} files` : ''
+    )
+  );
   logMissingReviews();
   await logUnmappedUsers();
+  logBrokenFiles();
 }
 
 extract().then(() => {
@@ -115,8 +145,15 @@ function logMissingReviews() {
   console.log(_(missingReviewKeys).map(key => reversePullRequests[key]).sort().join('\n'));
 }
 
+function logBrokenFiles() {
+  if (!args.download || !brokenFiles.length) return;
+  console.log(`\n${brokenFiles.length} files could not be downloaded:`);
+  console.log(brokenFiles.join('\n'));
+}
+
 async function extractSystem() {
-  const system = await db.get('system');
+  log('Extracting /system');
+  const system = await db.child('system').get();
   if (system.star && system.star !== '*' || system.bang && system.bang !== '!') {
     throw new Error('Bad or missing REVIEWABLE_ENCRYPTION_AES_KEY');
   }
@@ -128,6 +165,7 @@ async function extractSystem() {
 
 async function extractOrganizations() {
   if (!orgNames.length) return;
+  log('Extracting organizations');
   await forEachLimit(orgNames, 5, async org => {
     const organization = await db.child('organizations/:org', {org}).get();
     await writeItem(`organizations/${toKey(org)}`, organization);
@@ -137,6 +175,7 @@ async function extractOrganizations() {
 
 async function extractRepositories() {
   if (!repoNames.length) return;
+  log('Extracting repositories');
   await forEachLimit(repoNames, 10, async repoName => {
     const [owner, repo] = repoName.split('/');
     let repository = await db.child('repositories/:owner/:repo', {owner, repo}).get();
@@ -165,6 +204,7 @@ async function extractRepositories() {
 
 async function extractRules() {
   if (!repoNames.length) return;
+  log('Extracting rules');
   await forEachLimit(repoNames, 10, async repoName => {
     const [owner, repo] = repoName.split('/');
     const rule = await db.child('rules/:owner/:repo', {owner, repo}).get();
@@ -175,6 +215,7 @@ async function extractRules() {
 
 async function extractReviews() {
   if (!reviewKeys.length) return;
+  log('Extracting reviews');
   await forEachLimit(reviewKeys, 25, async reviewKey => {
     let review = await db.child('reviews/:reviewKey', {reviewKey}).get();
     if (review) {
@@ -184,12 +225,12 @@ async function extractReviews() {
       const archive = await db.child('archivedReviews/:reviewKey', {reviewKey}).get();
       if (archive) {
         review = JSON.parse(zlib.gunzipSync(Buffer.from(archive.payload, 'base64')).toString());
-        stripReview(review);
+        const placeholdersPresent = stripReview(review);
         if (identityUserMap) mapAllUserKeys(review);
         archive.payload =
           zlib.gzipSync(JSON.stringify(review), {level: zlib.constants.Z_BEST_COMPRESSION})
             .toString('base64');
-        await writeItem(`archivedReviews/${reviewKey}`, archive);
+        await writeItem(`archivedReviews/${reviewKey}`, archive, {placeholdersPresent});
       } else {
         missingReviewKeys.push(reviewKey);
       }
@@ -198,12 +239,33 @@ async function extractReviews() {
   });
 }
 
-function stripReview(review) {
+async function stripReview(review) {
+  let placeholderAdded = false;
   review.core = _.omit(review.core, 'lastSweepTimestamp');
   delete review.lastWebhook;
+  const downloadPromises = [];
   review.discussions = _.pickBy(review.discussions, discussion => {
     discussion.comments =
       _.pickBy(discussion.comments, (comment, commentKey) => !/^gh-/.test(commentKey));
+    if (uploadedFilesUrl) {
+      _.forEach(discussion.comments, comment => {
+        if (!comment.markdownBody) return;
+        const body = comment.markdownBody.replace(uploadedFilesUrlRegex, (match, host, rest) => {
+          const url = host + rest;
+          if (args.download && !downloadedFiles.has(url)) {
+            downloadedFiles.add(url);
+            const dest = path.join(args.download, path.dirname(rest.slice(1)));
+            downloadPromises.push(download(url, dest).catch(e => {
+              if (args.logging) log(`File download failed:\n${url}\n${e}`);
+              brokenFiles.push(url);
+            }));
+          }
+          return PLACEHOLDER_URL + rest;
+        });
+        if (body !== comment.markdownBody) placeholderAdded = true;
+        comment.markdownBody = body;
+      });
+    }
     return !_.isEmpty(discussion.comments);
   });
   if (_.isEmpty(review.discussions)) delete review.discussions;
@@ -219,10 +281,13 @@ function stripReview(review) {
     return !_.isEmpty(sentiment.comments);
   });
   if (_.isEmpty(review.sentiments)) delete review.sentiments;
+  if (!_.isEmpty(downloadPromises)) await Promise.all(downloadPromises);
+  return placeholderAdded;
 }
 
 async function extractLinemaps() {
   if (!reviewKeys.length) return;
+  log('Extracting linemaps');
   await forEachLimit(reviewKeys, 25, async reviewKey => {
     const linemap = await db.child('linemaps/:reviewKey', {reviewKey}).get();
     await writeItem(`linemaps/${reviewKey}`, linemap);
@@ -232,6 +297,7 @@ async function extractLinemaps() {
 
 async function extractFilemaps() {
   if (!reviewKeys.length) return;
+  log('Extracting filemaps');
   await forEachLimit(reviewKeys, 25, async reviewKey => {
     const filemap = await db.child('filemaps/:reviewKey', {reviewKey}).get();
     await writeItem(`filemaps/${reviewKey}`, filemap);
@@ -241,6 +307,7 @@ async function extractFilemaps() {
 
 async function extractUsers() {
   if (_.isEmpty(userMap)) return;
+  log('Extracting users');
   await forEachOfLimit(userMap, 25, async (newUserKey, oldUserKey) => {
     let user = await db.child('users/:oldUserKey', {oldUserKey}).get();
     user = _.omit(
@@ -266,10 +333,15 @@ async function extractUsers() {
   });
 }
 
-async function writeItem(key, value) {
+async function writeItem(key, value, flags) {
   if (value === undefined || value === null) return;
   value = mapAllUserKeys(value, key);
-  await out.write(`[${JSON.stringify(key)}, ${JSON.stringify(value)}]\n`);
+  if (flags) {
+    await out.write(
+      `[${JSON.stringify(key)}, ${JSON.stringify(value)}, ${JSON.stringify(flags)}]\n`);
+  } else {
+    await out.write(`[${JSON.stringify(key)}, ${JSON.stringify(value)}]\n`);
+  }
 }
 
 function mapAllUserKeys(object, context) {
@@ -306,3 +378,7 @@ function mapUserKey(userKey, context) {
 function toKey(value) {
   return NodeFire.escape(value);
 }
+
+function log(...params) {
+  if (args.logging) console.log('---', ...params);
+}

package/lib/derivedInfo.js

@@ -0,0 +1,28 @@
+export const uploadedFilesUrl = deriveUploadedFilesUrl();
+export const PLACEHOLDER_URL = 'https://REVIEWABLE_UPLOADED_FILES.URL';
+
+function deriveUploadedFilesUrl() {
+  if (process.env.REVIEWABLE_UPLOADED_FILES_URL) {
+    return process.env.REVIEWABLE_UPLOADED_FILES_URL.replace(/\/$/, '');
+  }
+  if (!process.env.REVIEWABLE_UPLOADS_PROVIDER) return;
+  switch (process.env.REVIEWABLE_UPLOADS_PROVIDER) {
+    case 'local':
+      return process.env.REVIEWABLE_HOST_URL + '/usercontent';
+
+    case 's3': {
+      let bucketUrl = 'https://s3.amazonaws.com/' + process.env.REVIEWABLE_S3_BUCKET;
+      if (process.env.AWS_REGION && process.env.AWS_REGION !== 'us-east-1') {
+        bucketUrl = bucketUrl.replace(/\/\/s3\./, '//s3-' + process.env.AWS_REGION + '.');
+      }
+      return bucketUrl;
+    }
+
+    case 'gcs':
+      return 'https://storage.googleapis.com/' + process.env.REVIEWABLE_GCS_BUCKET;
+
+    default:
+      throw new Error(
+        `Unknown REVIEWABLE_UPLOADS_PROVIDER: ${process.env.REVIEWABLE_UPLOADS_PROVIDER}`);
+  }
+}

package/lib/loadFirebase.js

@@ -1,6 +1,7 @@
 import _ from 'lodash';
 import * as fs from 'fs';
 import * as bytes from 'bytes';
+import * as path from 'path';
 import admin from 'firebase-admin';
 import nodefireModule from 'nodefire';
 import {patchFirebase} from 'firecrypt';
@@ -73,7 +74,8 @@ if (process.env.REVIEWABLE_ENCRYPTION_AES_KEY) {
     algorithm: 'aes-siv', key: process.env.REVIEWABLE_ENCRYPTION_AES_KEY,
     cacheSize: FIRECRYPT_CACHE_SIZE
   };
-  const specification = JSON.parse(fs.readFileSync('rules_firecrypt.json', 'utf8'));
+  const specification = JSON.parse(fs.readFileSync(
+    path.join(__dirname, '..', 'rules_firecrypt.json'), 'utf8'));
   admin.database().configureEncryption(options, specification);
 }
 

package/lib/tokens.js

@@ -0,0 +1,41 @@
+import crypto from 'crypto';
+import * as constants from 'constants';
+import _ from 'lodash';
+
+const keys = [];
+
+export async function fetchToken(userKey) {
+  const encryptedToken = await db.child('users/:userKey/core/gitHubToken', {userKey}).get();
+  if (!encryptedToken) throw new Error(`User ${userKey} not signed in to Reviewable`);
+  if (!/^rsa\d*:/.test(encryptedToken)) return encryptedToken;
+  const cipherText = Buffer.from(encryptedToken.replace(/^rsa\d:/, ''), 'base64');
+  if (!process.env.REVIEWABLE_ENCRYPTION_PRIVATE_KEYS) {
+    throw new Error(
+      `Unable to decrypt token for user ${userKey} without REVIEWABLE_ENCRYPTION_PRIVATE_KEYS`);
+  }
+  for (const key of keys) {
+    const token = crypto.privateDecrypt(key, cipherText).toString('utf8');
+    if (/^[ -~]+$/.test(token)) return token;
+  }
+  throw new Error(`Unable to decrypt token for user ${userKey} with any private key`);
+}
+
+
+if (process.env.REVIEWABLE_ENCRYPTION_PRIVATE_KEYS) {
+  _.forEach(process.env.REVIEWABLE_ENCRYPTION_PRIVATE_KEYS.split(','), pemKey => {
+    const key = crypto.createPrivateKey(normalizePrivateKey(pemKey));
+    keys.push({key, padding: constants.RSA_PKCS1_OAEP_PADDING, oaepHash: 'sha256'});
+  });
+}
+
+
+function normalizePrivateKey(pkcsKey) {
+  return pkcsKey.replace(
+    /-----BEGIN (.*?) KEY-----([\s\S]*?)-----END (\1) KEY-----/,
+    (match, keyType, contents) => {
+      return '-----BEGIN ' + keyType + ' KEY-----\n' +
+        contents.replace(/\\n|\s+/g, '').replace(/.{64}/g, '$&\n').replace(/\n*$/, '\n') +
+        '-----END ' + keyType + ' KEY-----\n';
+    }
+  );
+}

package/load_data.js

@@ -2,13 +2,17 @@
 
 import _ from 'lodash';
 import * as fs from 'fs';
+import * as zlib from 'zlib';
 import es from 'event-stream';
 import commandLineArgs from 'command-line-args';
 import getUsage from 'command-line-usage';
 import nodefireModule from 'nodefire';
+import Hubkit from 'hubkit';
 import {PromiseReadable} from 'promise-readable';
 import Pace from 'pace';
 import {Throttle} from 'stream-throttle';
+import {uploadedFilesUrl, PLACEHOLDER_URL} from './lib/derivedInfo.js';
+import {fetchToken} from './lib/tokens.js';
 
 const NodeFire = nodefireModule.default;
 
@@ -43,11 +47,21 @@ for (const property of ['input', 'admin']) {
 }
 
 if (!process.env.REVIEWABLE_ENCRYPTION_AES_KEY) {
-  console.log('WARNING: not encrypting uploaded data as REVIEWABLE_ENCRYPTION_AES_KEY not given');
+  console.warn('WARNING: not encrypting uploaded data as REVIEWABLE_ENCRYPTION_AES_KEY not given');
+}
+
+let placeholderUrlRegex, gh;
+if (uploadedFilesUrl) {
+  placeholderUrlRegex = new RegExp(_.escapeRegExp(PLACEHOLDER_URL), 'g');
+} else {
+  console.warn(
+    'WARNING: no REVIEWABLE_UPLOADS_PROVIDER or REVIEWABLE_UPLOADED_FILES_URL specified, ' +
+    'so not rewriting uploaded image URLs in comments.');
 }
 
 async function load() {
   await import('./lib/loadFirebase.js');
+  if (uploadedFilesUrl) gh = new Hubkit({token: await fetchToken(args.admin)});
 
   let sizeRead = 0;
   let fatalError;
@@ -84,8 +98,21 @@ load().then(() => {
 });
 
 
-async function processLine([key, value]) {
+async function processLine([key, value, flags]) {
+  if (uploadedFilesUrl && !_.isEmpty(value)) {
+    if (_.startsWith(key, 'reviews/')) {
+      await tweakReview(value);
+    } else if (_.startsWith(key, 'archivedReviews/') && flags?.placeholdersPresent) {
+      const review = JSON.parse(zlib.gunzipSync(Buffer.from(value.payload, 'base64')).toString());
+      await tweakReview(review);
+      value.payload =
+        zlib.gzipSync(JSON.stringify(review), {level: zlib.constants.Z_BEST_COMPRESSION})
+          .toString('base64');
+    }
+  }
+
   if (!_.isEmpty(value)) await db.child(key).update(value);
+
   if (_.startsWith(key, 'reviews/')) {
     const syncOptions = {
       userKey: args.admin, prNumber: value.core.pullRequestId,
@@ -103,3 +130,23 @@ async function processLine([key, value]) {
   }
 }
 
+async function tweakReview(review) {
+  const fullRepoName = `${review.core.ownerName}/${review.core.repoName}`;
+  const promises = [];
+  _.forEach(review.discussions, discussion => {
+    _.forEach(discussion.comments, comment => {
+      if (!comment.markdownBody) return;
+      const body = comment.markdownBody.replace(placeholderUrlRegex, uploadedFilesUrl);
+      if (body === comment.markdownBody) return;
+      comment.markdownBody = body;
+      promises.push(
+        gh.request('POST /markdown', {body: {
+          text: body, mode: 'gfm', context: fullRepoName
+        }}).then(htmlBody => {
+          comment.htmlBody = htmlBody;
+        })
+      );
+    });
+  });
+  if (promises.length) await Promise.all(promises);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "reviewable-enterprise-tools",
-  "version": "1.1.0",
+  "version": "1.3.1",
   "type": "module",
   "description": "Admin tools for Reviewable Enterprise",
   "bin": {
@@ -9,6 +9,8 @@
     "read": "./read.js"
   },
   "scripts": {
+    "extract_data": "node ./extract_data.js",
+    "load_data": "node ./load_data.js",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "engines": {
@@ -36,9 +38,11 @@
     "bytes": "^3.1.0",
     "command-line-args": "^5.1.1",
     "command-line-usage": "^6.1.0",
+    "download": "^8.0.0",
     "event-stream": "^4.0.1",
     "firebase-admin": "9.3.0",
     "firecrypt": "^2.0.9",
+    "hubkit": "^3.0.0",
     "lodash": "^4.17.20",
     "ms": "^2.1.2",
     "nodefire": "^3.0.0",