tspl-cli 0.0.0-stage → 0.1.1

package/src/analysis/js-parser.js

@@ -0,0 +1,473 @@
+'use strict'
+
+const { parse } = require('@babel/parser')
+const { isSecretValue } = require('./file-filter')
+
+const HTTP_METHODS = ['get', 'post', 'put', 'patch', 'delete', 'head', 'options', 'request']
+const HTTP_VERB = /^(get|post|put|patch|delete|head|options)$/i
+const HTTP_CLIENTS = /^\$?(http|httpclient|axios|api|apiservice|apiclient|client|request|instance)$/i
+const URL_LIKE = /^(\/|\{\}\/|https?:\/\/)/
+const EXPRESS_ROUTERS = /^(router|app|route|api)$/i
+const UI_ROUTE_KEYS = ['component', 'loadChildren', 'loadComponent', 'redirectTo', 'children']
+const SKIP_KEYS = new Set(['loc', 'start', 'end', 'extra', 'leadingComments', 'trailingComments', 'innerComments', 'comments', 'tokens'])
+const LABEL_ATTRIBUTES = /^(title|placeholder|label|text|header|headerTitle|tabBarLabel|accessibilityLabel|alt)$/
+const LIMIT = 500
+const LABEL_LIMIT = 150
+const VALUE_LIMIT = 200
+const TEXT_LIMIT = 100
+const COMMENT_LIMIT = 50
+const CONSTANT_NAME = /^[A-Z][A-Z0-9]*(_[A-Z0-9]+)+$|^[A-Z]{3,}$/
+const CODE_LIKE = /[;{}=]|=>|\(\)|^\s*(const|let|var|if|return|import|await)\b/
+
+const pluginsFor = ext => {
+  if (ext === '.ts') return ['typescript', 'decorators-legacy']
+  if (ext === '.tsx') return ['typescript', 'jsx', 'decorators-legacy']
+  return ['jsx', 'decorators-legacy']
+}
+
+const walk = (root, visit) => {
+  const stack = [root]
+  while (stack.length) {
+    const node = stack.pop()
+    visit(node)
+    for (const key of Object.keys(node)) {
+      if (SKIP_KEYS.has(key)) continue
+      const value = node[key]
+      if (Array.isArray(value)) {
+        for (let i = value.length - 1; i >= 0; i--) {
+          if (value[i] && typeof value[i].type === 'string') stack.push(value[i])
+        }
+      } else if (value && typeof value.type === 'string') {
+        stack.push(value)
+      }
+    }
+  }
+}
+
+const lineOf = node => (node.loc ? node.loc.start.line : null)
+
+const keyName = prop => {
+  if (!prop || !prop.key) return null
+  if (prop.key.type === 'Identifier') return prop.key.name
+  if (prop.key.type === 'StringLiteral') return prop.key.value
+  return null
+}
+
+const property = (object, name) =>
+  object.properties.find(
+    prop => (prop.type === 'ObjectProperty' || prop.type === 'ObjectMethod') && keyName(prop) === name,
+  )
+
+const text = node => {
+  if (!node) return null
+  if (node.type === 'StringLiteral') return node.value
+  if (node.type === 'TemplateLiteral') {
+    return node.quasis.map(q => q.value.cooked).join('{}')
+  }
+  if (node.type === 'BinaryExpression' && node.operator === '+') {
+    const left = text(node.left)
+    const right = text(node.right)
+    if (left === null && right === null) return null
+    return `${left === null ? '{}' : left}${right === null ? '{}' : right}`
+  }
+  return null
+}
+
+const lastName = node => {
+  if (!node) return null
+  if (node.type === 'Identifier') return node.name
+  if (node.type === 'ThisExpression') return 'this'
+  if (node.type === 'MemberExpression' || node.type === 'OptionalMemberExpression') {
+    return node.property.type === 'Identifier' ? node.property.name : null
+  }
+  return null
+}
+
+const memberNames = node => {
+  if (!node) return []
+  if (node.type === 'Identifier') return [node.name]
+  if (node.type === 'MemberExpression' && node.property.type === 'Identifier') {
+    return [...memberNames(node.object), node.property.name]
+  }
+  return []
+}
+
+const handlerName = route => {
+  const holder = ['config', 'options']
+    .map(key => property(route, key))
+    .find(prop => prop && prop.value && prop.value.type === 'ObjectExpression')
+  const handler = property(route, 'handler') || (holder && property(holder.value, 'handler'))
+  if (!handler || !handler.value) return null
+  const names = memberNames(handler.value).filter(name => name !== 'handler')
+  return names.length > 1 ? names[names.length - 1] : null
+}
+
+const labelText = value => {
+  const label = value.replace(/\s+/g, ' ').trim()
+  return label.length >= 2 && label.length <= 80 && /[a-z]/i.test(label) ? label : null
+}
+
+const literalValue = node => {
+  if (!node) return null
+  if (node.type === 'StringLiteral') return node.value
+  if (node.type === 'NumericLiteral' || node.type === 'BooleanLiteral') return String(node.value)
+  if (node.type === 'TemplateLiteral' && !node.expressions.length) return node.quasis.map(q => q.value.cooked).join('')
+  return null
+}
+
+const valueOf = (name, node) => {
+  const value = literalValue(node)
+  if (value === null || !value.trim() || value.length > 80 || isSecretValue(name, value)) return null
+  return value
+}
+
+const commentLines = comments =>
+  comments
+    .flatMap(comment => comment.value.split('\n'))
+    .map(line => line.replace(/^[\s*/#-]+/, '').replace(/\s+/g, ' ').trim())
+    .filter(line => line.length <= 120 && line.split(' ').length >= 3 && /[a-z]{3}/i.test(line) && !CODE_LIKE.test(line))
+
+const isModuleExports = node =>
+  node.type === 'MemberExpression' &&
+  node.object.type === 'Identifier' &&
+  node.object.name === 'module' &&
+  node.property.type === 'Identifier' &&
+  node.property.name === 'exports'
+
+const classInfo = (node, name) => ({
+  kind: 'class',
+  name: name || (node.id && node.id.name) || 'default',
+  line: lineOf(node),
+  methods: node.body.body
+    .filter(m => (m.type === 'ClassMethod' || m.type === 'ClassProperty') && keyName(m) && keyName(m) !== 'constructor')
+    .filter(m => m.type === 'ClassMethod' || (m.value && /Function/.test(m.value.type)))
+    .map(keyName)
+    .slice(0, 100),
+})
+
+const decoratorsOf = node =>
+  (node.decorators || [])
+    .map(d => d.expression)
+    .filter(e => e && e.type === 'CallExpression' && e.callee.type === 'Identifier')
+
+const topLevelSymbols = (program, strings) => {
+  const symbols = []
+  const addDeclaration = decl => {
+    if (!decl) return
+    if (decl.type === 'FunctionDeclaration' && decl.id) {
+      symbols.push({ kind: 'function', name: decl.id.name, line: lineOf(decl) })
+    } else if (decl.type === 'ClassDeclaration') {
+      symbols.push(classInfo(decl))
+    } else if (decl.type === 'VariableDeclaration') {
+      decl.declarations.forEach(d => {
+        if (d.id.type !== 'Identifier' || !d.init) return
+        if (d.init.type === 'StringLiteral') strings.set(d.id.name, d.init.value)
+        if (/Function/.test(d.init.type)) {
+          symbols.push({ kind: 'function', name: d.id.name, line: lineOf(d) })
+        } else if (d.init.type === 'ClassExpression') {
+          symbols.push(classInfo(d.init, d.id.name))
+        }
+      })
+    }
+  }
+
+  program.body.forEach(statement => {
+    if (statement.type === 'ExportNamedDeclaration' || statement.type === 'ExportDefaultDeclaration') {
+      addDeclaration(statement.declaration)
+    } else if (
+      statement.type === 'ExpressionStatement' &&
+      statement.expression.type === 'AssignmentExpression' &&
+      isModuleExports(statement.expression.left) &&
+      statement.expression.right.type === 'ClassExpression'
+    ) {
+      symbols.push(classInfo(statement.expression.right))
+    } else {
+      addDeclaration(statement)
+    }
+  })
+  return symbols
+}
+
+const extractJs = (code, ext) => {
+  let ast
+  try {
+    ast = parse(code, {
+      sourceType: 'unambiguous',
+      errorRecovery: true,
+      allowReturnOutsideFunction: true,
+      plugins: pluginsFor(ext),
+    })
+  } catch (err) {
+    return { parseError: true }
+  }
+
+  const strings = new Map()
+  const symbols = topLevelSymbols(ast.program, strings)
+  const imports = new Set()
+  const exports = new Set()
+  const routes = []
+  const uiRoutes = []
+  const schemas = []
+  const modelNames = []
+  const modelSchemas = new Map()
+  const schemaVariables = new Map()
+  const httpCalls = []
+  const components = []
+  const screens = []
+  const refs = new Set()
+  const labels = new Set()
+  const values = []
+  const texts = new Set()
+  const topLevel = new Set()
+  ast.program.body.forEach(statement => {
+    const declaration = statement.type === 'ExportNamedDeclaration' ? statement.declaration : statement
+    if (declaration && declaration.type === 'VariableDeclaration') declaration.declarations.forEach(d => topLevel.add(d))
+  })
+  const addValue = (name, node) => {
+    const value = valueOf(name, node)
+    if (value !== null && values.length < VALUE_LIMIT) values.push({ name, value, line: lineOf(node) })
+  }
+
+  walk(ast.program, node => {
+    switch (node.type) {
+      case 'VariableDeclarator':
+        if (node.id.type === 'Identifier' && node.init && /(New|Call)Expression/.test(node.init.type) && lastName(node.init.callee) === 'Schema') {
+          schemaVariables.set(node.init, node.id.name)
+        }
+        if (node.id.type === 'Identifier' && (topLevel.has(node) || CONSTANT_NAME.test(node.id.name))) {
+          addValue(node.id.name, node.init)
+        }
+        break
+      case 'ObjectProperty': {
+        const name = keyName(node)
+        if (name && CONSTANT_NAME.test(name)) addValue(name, node.value)
+        break
+      }
+      case 'StringLiteral':
+      case 'TemplateLiteral': {
+        const value = node.type === 'StringLiteral' ? node.value : text(node)
+        if (value && value.length >= 8 && value.length <= 120 && / [a-z]/i.test(value) && !URL_LIKE.test(value) && !isSecretValue(null, value) && texts.size < TEXT_LIMIT) {
+          texts.add(value.replace(/\s+/g, ' ').trim())
+        }
+        break
+      }
+      case 'Identifier':
+        if (node.name.length >= 4 && /[A-Z]/.test(node.name)) refs.add(node.name)
+        break
+      case 'JSXText': {
+        const label = labelText(node.value)
+        if (label) labels.add(label)
+        break
+      }
+      case 'JSXAttribute': {
+        const label = node.name.type === 'JSXIdentifier' && LABEL_ATTRIBUTES.test(node.name.name) &&
+          node.value && node.value.type === 'StringLiteral' && labelText(node.value.value)
+        if (label) labels.add(label)
+        break
+      }
+      case 'ImportDeclaration':
+        imports.add(node.source.value)
+        break
+      case 'ExportAllDeclaration':
+        imports.add(node.source.value)
+        exports.add('*')
+        break
+      case 'ExportNamedDeclaration':
+        if (node.source) imports.add(node.source.value)
+        if (node.declaration) {
+          const d = node.declaration
+          if (d.id) exports.add(d.id.name)
+          if (d.declarations) d.declarations.forEach(v => v.id.name && exports.add(v.id.name))
+        }
+        node.specifiers.forEach(s => exports.add(s.exported.name || s.exported.value))
+        break
+      case 'ExportDefaultDeclaration':
+        exports.add('default')
+        break
+      case 'AssignmentExpression': {
+        const left = node.left
+        if (isModuleExports(left)) {
+          if (node.right.type === 'ObjectExpression') {
+            node.right.properties.forEach(p => keyName(p) && exports.add(keyName(p)))
+          } else {
+            exports.add('default')
+          }
+        } else if (
+          left.type === 'MemberExpression' &&
+          left.property.type === 'Identifier' &&
+          ((left.object.type === 'Identifier' && left.object.name === 'exports') || isModuleExports(left.object))
+        ) {
+          exports.add(left.property.name)
+        }
+        break
+      }
+      case 'CallExpression':
+      case 'OptionalCallExpression':
+      case 'NewExpression': {
+        const callee = node.callee
+        const first = node.arguments[0]
+        if (callee.type === 'Identifier' && callee.name === 'require' && first && first.type === 'StringLiteral') {
+          imports.add(first.value)
+        }
+        if (callee.type === 'Import' && first && first.type === 'StringLiteral') {
+          imports.add(first.value)
+        }
+        if (lastName(callee) === 'Schema' && first && first.type === 'ObjectExpression') {
+          const options = node.arguments[1]
+          const collection = options && options.type === 'ObjectExpression' && property(options, 'collection')
+          schemas.push({
+            variable: schemaVariables.get(node) || null,
+            fields: first.properties.map(keyName).filter(Boolean).slice(0, 200),
+            collection: collection ? text(collection.value) || strings.get(collection.value.name) || null : null,
+            line: lineOf(node),
+          })
+        }
+        if (callee.type === 'MemberExpression' && callee.property.type === 'Identifier') {
+          const method = callee.property.name
+          const owner = lastName(callee.object)
+          if (method === 'model' && first) {
+            const name = first.type === 'StringLiteral' ? first.value : strings.get(first.name)
+            if (name) modelNames.push(name)
+            const schema = node.arguments[1]
+            if (name && schema && schema.type === 'Identifier') modelSchemas.set(schema.name, name)
+          }
+          if (HTTP_METHODS.includes(method) && owner && HTTP_CLIENTS.test(owner)) {
+            const url = text(first)
+            if (url !== null) httpCalls.push({ method: method.toUpperCase(), url, line: lineOf(node) })
+          }
+          if (HTTP_METHODS.includes(method) && owner && EXPRESS_ROUTERS.test(owner) && first && first.type === 'StringLiteral' && first.value.startsWith('/')) {
+            routes.push({ method: method.toUpperCase(), path: first.value, line: lineOf(node) })
+          }
+        }
+        const second = node.arguments[1]
+        if (first && first.type === 'StringLiteral' && HTTP_VERB.test(first.value) && second) {
+          const url = text(second)
+          if (url !== null && URL_LIKE.test(url)) {
+            httpCalls.push({ method: first.value.toUpperCase(), url, line: lineOf(node) })
+          }
+        }
+        if (callee.type === 'Identifier' && callee.name === 'fetch' && first) {
+          const url = text(first)
+          const options = node.arguments[1]
+          const method = options && options.type === 'ObjectExpression' && property(options, 'method')
+          if (url !== null) {
+            httpCalls.push({ method: method ? (text(method.value) || 'GET').toUpperCase() : 'GET', url, line: lineOf(node) })
+          }
+        }
+        break
+      }
+      case 'ObjectExpression': {
+        const methodProp = property(node, 'method')
+        const pathProp = property(node, 'path')
+        const urlProp = property(node, 'url')
+        if (urlProp && urlProp.value && !pathProp) {
+          const url = text(urlProp.value)
+          const verb = methodProp && methodProp.value ? text(methodProp.value) : 'GET'
+          if (url !== null && URL_LIKE.test(url) && verb && HTTP_VERB.test(verb)) {
+            httpCalls.push({ method: verb.toUpperCase(), url, line: lineOf(node) })
+          }
+        } else if (methodProp && pathProp && pathProp.value) {
+          const path = text(pathProp.value)
+          const value = methodProp.value
+          const methods = value.type === 'ArrayExpression'
+            ? value.elements.map(text).filter(Boolean)
+            : [text(value)].filter(Boolean)
+          if (path !== null && methods.length) {
+            routes.push({ method: methods.join(',').toUpperCase(), path, handler: handlerName(node), line: lineOf(node) })
+          }
+        } else if (pathProp && pathProp.value && UI_ROUTE_KEYS.some(key => property(node, key))) {
+          const path = text(pathProp.value)
+          const component = property(node, 'component')
+          const lazy = property(node, 'loadChildren') || property(node, 'loadComponent')
+          let target = null
+          if (lazy) {
+            walk(lazy.value, inner => {
+              if (!target && inner.type === 'CallExpression' && inner.callee.type === 'Import' && inner.arguments[0]) {
+                target = text(inner.arguments[0])
+              }
+            })
+          }
+          if (path !== null) {
+            uiRoutes.push({
+              path,
+              component: component && component.value.type === 'Identifier' ? component.value.name : null,
+              lazy: target,
+              line: lineOf(node),
+            })
+          }
+        }
+        break
+      }
+      case 'ClassDeclaration': {
+        decoratorsOf(node).forEach(decorator => {
+          const kind = decorator.callee.name
+          if (!['Component', 'Injectable', 'NgModule', 'Directive', 'Pipe'].includes(kind)) return
+          const config = decorator.arguments[0]
+          const selector = config && config.type === 'ObjectExpression' && property(config, 'selector')
+          const template = config && config.type === 'ObjectExpression' && property(config, 'templateUrl')
+          components.push({
+            kind: kind === 'Injectable' ? 'service' : kind.toLowerCase(),
+            name: node.id ? node.id.name : 'default',
+            selector: selector ? text(selector.value) : null,
+            template: template ? text(template.value) : null,
+            line: lineOf(node),
+          })
+        })
+        break
+      }
+      case 'JSXOpeningElement': {
+        const name = node.name
+        if (name.type === 'JSXMemberExpression' && name.property.name === 'Screen') {
+          const attr = attrName => node.attributes.find(a => a.type === 'JSXAttribute' && a.name.name === attrName)
+          const screenName = attr('name')
+          const component = attr('component')
+          if (screenName && screenName.value && screenName.value.type === 'StringLiteral') {
+            screens.push({
+              name: screenName.value.value,
+              component: component && component.value && component.value.expression && component.value.expression.type === 'Identifier'
+                ? component.value.expression.name
+                : null,
+              line: lineOf(node),
+            })
+          }
+        }
+        break
+      }
+      default:
+        break
+    }
+  })
+
+  const byPosition = !modelSchemas.size && schemas.length === modelNames.length
+  const models = schemas.map(({ variable, ...schema }, i) => ({
+    name: modelSchemas.get(variable) || (schemas.length === 1 || byPosition ? modelNames[i] || null : null),
+    ...(variable ? { variable } : {}),
+    ...schema,
+  }))
+  modelNames
+    .filter(name => !models.some(m => m.name === name))
+    .forEach(name => models.push({ name, fields: [], collection: null, line: null }))
+
+  const defined = new Set(symbols.map(symbol => symbol.name))
+
+  return {
+    imports: [...imports].slice(0, LIMIT),
+    exports: [...exports].slice(0, LIMIT),
+    symbols: symbols.slice(0, LIMIT),
+    refs: [...refs].filter(name => !defined.has(name)).slice(0, LIMIT),
+    labels: [...labels].slice(0, LABEL_LIMIT),
+    values,
+    texts: [...texts].filter(value => !labels.has(value)),
+    comments: [...new Set(commentLines(ast.comments || []))].slice(0, COMMENT_LIMIT),
+    routes: routes.slice(0, LIMIT),
+    uiRoutes: uiRoutes.slice(0, LIMIT),
+    models: models.slice(0, LIMIT),
+    httpCalls: httpCalls.slice(0, LIMIT),
+    components: components.slice(0, LIMIT),
+    screens: screens.slice(0, LIMIT),
+  }
+}
+
+module.exports = {
+  extractJs,
+}

package/src/analysis/py-parser.js

@@ -0,0 +1,66 @@
+'use strict'
+
+const { isSecretValue } = require('./file-filter')
+
+const LIMIT = 500
+
+const extractPy = code => {
+  const imports = new Set()
+  const symbols = []
+  const routes = []
+  const values = []
+  const comments = new Set()
+  let currentClass = null
+
+  code.split('\n').forEach((raw, index) => {
+    const line = index + 1
+    const fromImport = raw.match(/^\s*from\s+([\w.]+)\s+import\s/)
+    const plainImport = raw.match(/^\s*import\s+([\w.]+)/)
+    const topDef = raw.match(/^(?:async\s+)?def\s+(\w+)/)
+    const topClass = raw.match(/^class\s+(\w+)/)
+    const method = raw.match(/^\s+(?:async\s+)?def\s+(\w+)/)
+    const route = raw.match(/^\s*@\w+\.(route|get|post|put|patch|delete)\(\s*['"]([^'"]+)['"]/)
+
+    const constant = raw.match(/^([A-Za-z_]\w*)\s*=\s*(?:(['"])(.*?)\2|(-?\d+(?:\.\d+)?|True|False))\s*(?:#.*)?$/)
+    const comment = raw.match(/^\s*#\s*(.+)$/)
+    if (constant) {
+      const value = constant[3] !== undefined ? constant[3] : constant[4]
+      if (value.trim() && value.length <= 80 && !isSecretValue(constant[1], value)) values.push({ name: constant[1], value, line })
+    }
+    if (comment && comment[1].split(' ').length >= 3 && comment[1].length <= 120) comments.add(comment[1].trim())
+
+    if (fromImport) imports.add(fromImport[1])
+    else if (plainImport) imports.add(plainImport[1])
+
+    if (topClass) {
+      currentClass = { kind: 'class', name: topClass[1], line, methods: [] }
+      symbols.push(currentClass)
+    } else if (topDef) {
+      currentClass = null
+      symbols.push({ kind: 'function', name: topDef[1], line })
+    } else if (method && currentClass && method[1] !== '__init__') {
+      currentClass.methods.push(method[1])
+    }
+
+    if (route) {
+      routes.push({
+        method: route[1] === 'route' ? 'GET' : route[1].toUpperCase(),
+        path: route[2],
+        line,
+      })
+    }
+  })
+
+  return {
+    imports: [...imports].slice(0, LIMIT),
+    exports: [],
+    symbols: symbols.slice(0, LIMIT),
+    routes: routes.slice(0, LIMIT),
+    values: values.slice(0, 200),
+    comments: [...comments].slice(0, 50),
+  }
+}
+
+module.exports = {
+  extractPy,
+}

package/src/analysis/repo-index.js

@@ -0,0 +1,107 @@
+'use strict'
+
+const Path = require('path')
+const { resolveRef, listFiles, fileHistory, readBlobs } = require('./git-reader')
+const { classify } = require('./file-filter')
+const { PARSER_VERSION, extractFacts } = require('./extract')
+
+const BATCH = 500
+const CONFIG_FILES = ['package.json', 'tsconfig.json', 'jsconfig.json']
+
+const parseJson = code => {
+  try {
+    return JSON.parse(code)
+  } catch (err) {
+    try {
+      return JSON.parse(
+        code
+          .replace(/("(?:\\.|[^"\\])*")|\/\*[\s\S]*?\*\/|\/\/[^\n]*/g, (match, string) => string || '')
+          .replace(/,(\s*[}\]])/g, '$1'),
+      )
+    } catch (inner) {
+      return null
+    }
+  }
+}
+
+const moduleAliases = pkg =>
+  Object.entries((pkg && pkg._moduleAliases) || {})
+    .filter(([, to]) => typeof to === 'string')
+    .map(([from, to]) => ({ from, to: Path.posix.normalize(to), wildcard: false }))
+
+const pathAliases = tsconfig => {
+  const options = (tsconfig && tsconfig.compilerOptions) || {}
+  const base = options.baseUrl || '.'
+  return Object.entries(options.paths || {})
+    .filter(([, targets]) => Array.isArray(targets) && typeof targets[0] === 'string')
+    .map(([from, targets]) => ({
+      from: from.replace(/\*$/, ''),
+      to: Path.posix.normalize(Path.posix.join(base, targets[0].replace(/\*$/, ''))).replace(/^\.$/, ''),
+      wildcard: from.endsWith('*'),
+    }))
+}
+
+const readConfig = async (dir, files) => {
+  const byPath = new Map(files.map(file => [file.path, file.blobHash]))
+  const wanted = CONFIG_FILES.filter(name => byPath.has(name))
+  const blobs = wanted.length ? await readBlobs(dir, wanted.map(name => byPath.get(name))) : new Map()
+  const json = name => (byPath.has(name) ? parseJson(blobs.get(byPath.get(name)) || '') : null)
+  const pkg = json('package.json')
+  return {
+    packageName: (pkg && pkg.name) || null,
+    packageMain: (pkg && typeof pkg.main === 'string' && pkg.main) || null,
+    aliases: [...moduleAliases(pkg), ...pathAliases(json('tsconfig.json') || json('jsconfig.json'))],
+  }
+}
+
+const buildRepoIndex = async (repo, previous) => {
+  const branch = repo.defaultBaseBranch
+  const { ref, commitSha, date } = resolveRef(repo.path, branch)
+  const reusable = previous && previous.parserVersion === PARSER_VERSION
+  if (reusable && previous.commitSha === commitSha) {
+    return { ...previous, name: repo.name, newlyParsed: 0 }
+  }
+
+  const files = listFiles(repo.path, ref).map(file => ({
+    ...file,
+    status: classify(file.path, file.size),
+  }))
+  const known = reusable ? previous.facts : {}
+  const facts = {}
+  const toParse = []
+  files
+    .filter(file => file.status === 'parsed')
+    .forEach(file => {
+      if (known[file.blobHash]) {
+        facts[file.blobHash] = known[file.blobHash]
+      } else if (!facts[file.blobHash]) {
+        facts[file.blobHash] = null
+        toParse.push(file)
+      }
+    })
+
+  for (let i = 0; i < toParse.length; i += BATCH) {
+    const batch = toParse.slice(i, i + BATCH)
+    const blobs = await readBlobs(repo.path, batch.map(file => file.blobHash))
+    batch.forEach(file => {
+      facts[file.blobHash] = extractFacts(file.path, blobs.get(file.blobHash) || '')
+    })
+  }
+
+  return {
+    name: repo.name,
+    branch,
+    commitSha,
+    date,
+    parserVersion: PARSER_VERSION,
+    ...(await readConfig(repo.path, files)),
+    files,
+    facts,
+    history: fileHistory(repo.path, ref, new Set(files.filter(file => file.status === 'parsed').map(file => file.path))),
+    newlyParsed: toParse.length,
+  }
+}
+
+module.exports = {
+  buildRepoIndex,
+}