tspl-cli 0.0.0-stage → 0.1.1

package/src/find/search.js

@@ -0,0 +1,473 @@
+'use strict'
+
+const Path = require('path').posix
+const { words, terms, splitTerms, identifier, editDistance } = require('./text')
+
+const K1 = 1.2
+const B = 0.75
+const SPREAD = 0.35
+const DETAIL_WEIGHT = 0.6
+const SEEDS = 40
+const CHAIN_FILES = 12
+const TEST_FILE = /(^|[./_-])(spec|test|tests)([./_-]|$)/
+
+const addField = (doc, value, weight, label) => {
+  terms(value).forEach(term => {
+    doc.tf.set(term, (doc.tf.get(term) || 0) + weight)
+    doc.length += weight
+    if (!doc.why.has(term)) doc.why.set(term, `${label}: ${value}`)
+  })
+}
+
+const documentFor = (file, repoName) => {
+  const facts = file.facts
+  const doc = { tf: new Map(), why: new Map(), length: 0 }
+  const name = Path.basename(file.path).replace(/\.[^.]+$/, '')
+  addField(doc, name, 3, 'file')
+  addField(doc, Path.dirname(file.path), 1.5, 'folder')
+  doc.place = new Set([...terms(name), ...terms(Path.basename(Path.dirname(file.path)))])
+  addField(doc, repoName, 0.5, 'repo')
+  ;(facts.symbols || []).forEach(symbol => {
+    addField(doc, symbol.name, 2.5, symbol.kind)
+    ;(symbol.methods || []).forEach(method => addField(doc, method, 1.5, 'method'))
+  })
+  ;(facts.exports || []).forEach(value => addField(doc, value, 1.5, 'export'))
+  ;(facts.routes || []).forEach(route => {
+    addField(doc, route.path, 2, `route ${route.method}`)
+    if (route.handler) addField(doc, route.handler, 1.5, 'handler')
+  })
+  ;(facts.refs || []).forEach(name => {
+    const term = identifier(name)
+    if (!term) return
+    doc.tf.set(term, (doc.tf.get(term) || 0) + 0.5)
+    doc.length += 0.5
+    if (!doc.why.has(term)) doc.why.set(term, `mentions: ${name}`)
+  })
+  ;(facts.models || []).forEach(model => {
+    addField(doc, model.name, 3, 'model')
+    addField(doc, model.variable, 2, 'schema')
+    addField(doc, model.collection, 2, 'collection')
+    ;(model.fields || []).forEach(field => addField(doc, field, 0.7, 'field'))
+  })
+  ;(facts.components || []).forEach(component => {
+    addField(doc, component.name, 2.5, component.kind)
+    addField(doc, component.selector, 2, 'selector')
+  })
+  ;(facts.uiRoutes || []).forEach(route => addField(doc, route.path, 2, 'page route'))
+  ;(facts.screens || []).forEach(screen => addField(doc, screen.name, 2.5, 'screen'))
+  ;(facts.httpCalls || []).forEach(call => addField(doc, call.url, 1, `calls ${call.method}`))
+  ;(facts.labels || []).forEach(label => addField(doc, label, 1.5, 'text'))
+  ;(facts.tags || []).forEach(tag => addField(doc, tag, 0.8, 'uses'))
+  ;(facts.values || []).forEach(value => {
+    addField(doc, value.name, 2, `value ${value.name} = ${value.value}`)
+    if (/[a-z]{3}/i.test(value.value)) addField(doc, value.value, 1, `value ${value.name} = ${value.value}`)
+  })
+  ;(facts.texts || []).forEach(value => addField(doc, value, 1, 'message'))
+  ;(facts.comments || []).forEach(value => addField(doc, value, 0.8, 'comment'))
+  ;(file.history || []).forEach(message => addField(doc, message, 1, 'past change'))
+  return doc
+}
+
+const createSearch = graph => {
+  const docs = graph.files.map(file => documentFor(file, file.repo))
+  const df = new Map()
+  docs.forEach(doc => doc.tf.forEach((_, term) => df.set(term, (df.get(term) || 0) + 1)))
+  const averageLength = docs.reduce((sum, doc) => sum + doc.length, 0) / (docs.length || 1)
+  const idf = term => {
+    const n = df.get(term) || 0
+    return Math.log(1 + (docs.length - n + 0.5) / (n + 0.5))
+  }
+  const vocabulary = [...df.keys()]
+  const suffix = path => path.split('/').slice(-2).join('/')
+  const bySuffix = new Map()
+  graph.files.forEach((file, id) => {
+    const key = suffix(file.path)
+    if (!bySuffix.has(key)) bySuffix.set(key, [])
+    bySuffix.get(key).push(id)
+  })
+  const copiesOf = id => (bySuffix.get(suffix(graph.files[id].path)) || [])
+    .filter(other => graph.files[other].repo !== graph.files[id].repo)
+    .map(other => `${graph.files[other].repo}/${graph.files[other].path}`)
+
+  const expandQuery = text => {
+    const { single, joined, original } = splitTerms(text)
+    const known = []
+    const unknown = []
+    const add = (term, weight, typo) => {
+      if (!known.some(q => q.term === term)) known.push({ term, weight, typo })
+    }
+    ;[...new Set(single)].forEach(term => {
+      if (df.has(term)) {
+        add(term, 1)
+        return
+      }
+      const max = term.length >= 8 ? 2 : term.length >= 4 ? 1 : 0
+      const close = max
+        ? vocabulary.filter(word => word[0] === term[0] && editDistance(word, term, max) <= max).slice(0, 3)
+        : []
+      if (close.length) close.forEach(word => add(word, 0.7, original.get(term)))
+      else unknown.push(original.get(term))
+    })
+    joined.filter(term => df.has(term)).forEach(term => add(term, 1))
+    const fixed = term => (known.find(q => q.typo === original.get(term)) || { term }).term
+    for (let i = 0; i < single.length - 1; i++) {
+      const pair = fixed(single[i]) + fixed(single[i + 1])
+      if (pair !== single[i] + single[i + 1] && df.has(pair)) add(pair, 0.7)
+    }
+    const used = [...new Set(single)]
+      .filter(term => df.has(term))
+      .map(term => original.get(term))
+    const corrected = known.filter(q => q.typo).map(q => `${q.typo} → ${q.term}`)
+    return { known, unknown, used: [...used, ...corrected] }
+  }
+
+  const lexical = known => {
+    const scores = new Map()
+    const total = known.reduce((sum, q) => sum + idf(q.term) * q.weight, 0) || 1
+    docs.forEach((doc, id) => {
+      let score = 0
+      let covered = 0
+      known.forEach(q => {
+        const tf = doc.tf.get(q.term)
+        if (!tf) return
+        const weight = idf(q.term) * q.weight
+        score += weight * ((tf * (K1 + 1)) / (tf + K1 * (1 - B + (B * doc.length) / averageLength)))
+        covered += weight
+      })
+      if (score > 0) scores.set(id, { score: score * (0.3 + covered / total) ** 2, covered: covered / total })
+    })
+    return scores
+  }
+
+  const degree = id => graph.out[id].length + graph.into[id].length
+
+  const find = (text, options = {}) => {
+    const limit = options.limit || 15
+    const { known, unknown, used } = expandQuery(text)
+    if (options.title) {
+      const title = expandQuery(options.title).known.map(q => q.term)
+      if (title.length) {
+        const inTitle = new Set(title)
+        known.forEach(q => {
+          if (!inTitle.has(q.term)) q.weight *= DETAIL_WEIGHT
+        })
+      }
+    }
+    const lex = lexical(known)
+    const top = Math.max(0, ...[...lex.values()].map(item => item.score)) || 1
+    const final = new Map()
+    const via = new Map()
+    lex.forEach((item, id) => final.set(id, item.score / top))
+
+    const seeds = [...lex.entries()].sort((a, b) => b[1].score - a[1].score).slice(0, SEEDS)
+    seeds.forEach(([id, item]) => {
+      const strength = item.score / top
+      const spread = (edge, other, direction) => {
+        const own = lex.has(other) ? lex.get(other).score / top : 0
+        const bonus = strength * edge.weight * SPREAD * (1 + own) / Math.log2(2 + degree(other) / 4)
+        if (bonus < 0.02) return
+        final.set(other, (final.get(other) || 0) + bonus)
+        const current = via.get(other)
+        if (!current || current.bonus < bonus) via.set(other, { bonus, from: id, edge, direction })
+      }
+      graph.out[id].forEach(edge => spread(edge, edge.to, 'out'))
+      graph.into[id].forEach(edge => spread(edge, edge.from, 'in'))
+    })
+
+    let sorted = [...final.entries()].sort((a, b) => b[1] - a[1])
+    if (options.repos && options.repos.length) {
+      const chosen = new Set(options.repos)
+      const anchors = new Set(sorted.filter(([id]) => chosen.has(graph.files[id].repo)).slice(0, 20).map(([id]) => id))
+      const linked = id => graph.out[id].some(edge => anchors.has(edge.to)) || graph.into[id].some(edge => anchors.has(edge.from))
+      sorted = sorted.filter(([id]) => chosen.has(graph.files[id].repo) || linked(id))
+    }
+    const ranked = sorted.slice(0, limit).map(([id, score]) => ({ id, score }))
+
+    const titleTerms = options.title ? expandQuery(options.title).known.map(q => q.term) : []
+    const choices = choicesFor(graph, docs, final, titleTerms, options.repos)
+
+    const queryTerms = new Set(known.map(q => q.term))
+    const results = ranked.map(({ id, score }) => {
+      const file = graph.files[id]
+      const doc = docs[id]
+      const matched = known.filter(q => doc.tf.has(q.term)).sort((a, b) => idf(b.term) - idf(a.term))
+      const why = [...new Set(matched.map(q => doc.why.get(q.term)))].slice(0, 3)
+      const link = via.get(id)
+      if (link) {
+        const other = graph.files[link.from]
+        const verb = {
+          'calls-api': link.direction === 'out' ? 'is called by' : 'calls',
+          imports: link.direction === 'out' ? 'is imported by' : 'imports',
+          uses: link.direction === 'out' ? 'is used by' : 'uses',
+          template: 'is the template/code of',
+          renders: link.direction === 'out' ? 'is shown inside' : 'shows',
+          'same-value': `sets the same ${link.edge.detail} as`,
+        }[link.edge.type]
+        const where = other.repo === file.repo ? Path.basename(other.path) : `${other.repo}/${other.path}`
+        why.push(`${verb} ${where}${link.edge.detail && link.edge.type === 'calls-api' ? ` (${link.edge.detail})` : ''}`)
+      }
+      return {
+        id,
+        repo: file.repo,
+        path: file.path,
+        score: Number(score.toFixed(3)),
+        why,
+        symbols: symbolsFor(file, queryTerms),
+        similar: similarFiles(graph, file, ranked),
+        copies: copiesOf(id),
+      }
+    })
+
+    return {
+      query: text,
+      terms: used,
+      unknown,
+      files: results,
+      chains: chainsFor(graph, results),
+      areas: areasFor(graph, results),
+      choices,
+      searchSteps: searchStepsFor(graph, results, new Set(known.map(q => q.term))),
+      confidence: confidenceFor(known, unknown, lex, results, graph, top),
+      placement: placementFor(graph, results),
+    }
+  }
+
+  return { find }
+}
+
+const symbolsFor = (file, queryTerms) => {
+  const hits = []
+  const check = (name, kind, line) => {
+    const matched = terms(name).filter(term => queryTerms.has(term)).length
+    if (matched) hits.push({ name, kind, line: line || null, matched })
+  }
+  ;(file.facts.symbols || []).forEach(symbol => {
+    check(symbol.name, symbol.kind, symbol.line)
+    ;(symbol.methods || []).forEach(method => check(method, `method of ${symbol.name}`))
+  })
+  ;(file.facts.routes || []).forEach(route => check(route.path, `route ${route.method}`, route.line))
+  ;(file.facts.models || []).forEach(model => check(model.name || 'model', 'model', model.line))
+  return hits
+    .sort((a, b) => b.matched - a.matched)
+    .slice(0, 4)
+    .map(({ name, kind, line }) => ({ name, kind, line }))
+}
+
+const similarFiles = (graph, file, ranked) => {
+  const shown = new Set(ranked.map(item => item.id))
+  const name = Path.basename(file.path).replace(/\..*$/, '')
+  const mine = new Set(terms(name))
+  if (mine.size < 2 || TEST_FILE.test(file.path)) return []
+  const folder = Path.dirname(file.path)
+  return graph.files
+    .filter(other =>
+      other.repoId === file.repoId && !shown.has(other.id) && Path.dirname(other.path) === folder &&
+      Path.basename(other.path).replace(/\..*$/, '') !== name && !TEST_FILE.test(other.path))
+    .map(other => {
+      const theirs = terms(Path.basename(other.path).replace(/\..*$/, ''))
+      return { path: other.path, shared: theirs.filter(term => mine.has(term)).length / Math.max(mine.size, theirs.length) }
+    })
+    .filter(other => other.shared >= 0.5)
+    .sort((a, b) => b.shared - a.shared)
+    .slice(0, 3)
+    .map(other => other.path)
+}
+
+const searchStepsFor = (graph, results, queryTerms) => {
+  const steps = new Map()
+  const add = (pattern, why) => {
+    if (pattern && pattern.length >= 3 && !steps.has(pattern)) steps.set(pattern, why)
+  }
+  results.slice(0, 8).forEach(result => {
+    const facts = graph.files[result.id].facts
+    ;(facts.values || []).forEach(value => {
+      if (terms(value.name).filter(term => queryTerms.has(term)).length < 2) return
+      add(value.name, `constant in ${Path.basename(result.path)}`)
+      if (value.value.length >= 3 && !/^\d{1,3}$/.test(value.value)) add(`'${value.value}'`, `value of ${value.name}`)
+    })
+    ;(facts.symbols || []).forEach(symbol => {
+      ;[symbol.name, ...(symbol.methods || [])].forEach(name => {
+        if (terms(name).filter(term => queryTerms.has(term)).length >= 2) add(name, `defined in ${Path.basename(result.path)}`)
+      })
+    })
+  })
+  return [...steps.entries()].slice(0, 6).map(([pattern, why]) => ({ pattern, why }))
+}
+
+const AREA_FILES = 12
+const AREA_LINKS = new Set(['calls-api', 'template', 'renders', 'same-value', 'imports', 'uses'])
+const HUB = 15
+
+const areaName = path => {
+  const base = Path.basename(path).split('.')[0]
+  const folder = Path.basename(Path.dirname(path))
+  return words(base === 'index' ? folder : base).join(' ')
+}
+
+const areasFor = (graph, results) => {
+  const top = results.slice(0, AREA_FILES).filter(result => !TEST_FILE.test(result.path))
+  const ids = new Set(top.map(result => result.id))
+  const parent = new Map(top.map(result => [result.id, result.id]))
+  const root = id => (parent.get(id) === id ? id : root(parent.get(id)))
+  const hub = id => graph.into[id].length > HUB
+  top.forEach(result => {
+    graph.out[result.id].forEach(edge => {
+      if (!ids.has(edge.to) || !AREA_LINKS.has(edge.type)) return
+      if ((edge.type === 'imports' || edge.type === 'uses') && (hub(edge.to) || hub(edge.from))) return
+      parent.set(root(edge.from), root(edge.to))
+    })
+  })
+  const groups = new Map()
+  top.forEach(result => {
+    const key = root(result.id)
+    if (!groups.has(key)) groups.set(key, [])
+    groups.get(key).push(result)
+  })
+  return [...groups.values()]
+    .map(files => ({
+      name: areaName(files[0].path),
+      repo: files[0].repo,
+      score: files[0].score,
+      files: files.map(file => `${file.repo}/${file.path}`),
+    }))
+    .sort((a, b) => b.score - a.score)
+}
+
+const CHOICES = 4
+
+const choicesFor = (graph, docs, scores, titleTerms, repos) => {
+  const chosen = repos && repos.length ? new Set(repos) : null
+  const choices = []
+  titleTerms.forEach(term => {
+    const folders = new Map()
+    docs.forEach((doc, id) => {
+      const file = graph.files[id]
+      if (!doc.place.has(term) || TEST_FILE.test(file.path) || (chosen && !chosen.has(file.repo))) return
+      const key = `${file.repo}/${Path.dirname(file.path)}`
+      const entry = folders.get(key) || { repo: file.repo, folder: Path.dirname(file.path), score: 0, files: [] }
+      const score = scores.get(id) || 0
+      entry.score += score
+      entry.files.push({ path: `${file.repo}/${file.path}`, score })
+      folders.set(key, entry)
+    })
+    const best = [...folders.values()].sort((a, b) => b.score - a.score)[0]
+    if (!best || !best.score) return
+    choices.push({
+      term,
+      name: words(Path.basename(best.folder)).join(' ') || best.repo,
+      repo: best.repo,
+      score: best.score,
+      files: best.files.sort((a, b) => b.score - a.score).slice(0, 3).map(file => file.path),
+    })
+  })
+  const seen = new Set()
+  return choices
+    .sort((a, b) => b.score - a.score)
+    .filter(choice => !seen.has(choice.files[0]) && seen.add(choice.files[0]))
+    .slice(0, CHOICES)
+}
+
+const chainsFor = (graph, results) => {
+  const inSet = new Set(results.slice(0, CHAIN_FILES).filter(result => !TEST_FILE.test(result.path)).map(result => result.id))
+  const next = id =>
+    [...graph.out[id].filter(edge => edge.type !== 'same-value'), ...graph.into[id].filter(edge => edge.type === 'template')]
+      .map(edge => ({ edge, to: edge.from === id ? edge.to : edge.from }))
+      .filter(step => inSet.has(step.to))
+  const hasIncoming = id =>
+    graph.into[id].some(edge => inSet.has(edge.from) && edge.type !== 'template' && edge.type !== 'same-value')
+  const chains = []
+  const walk = (id, path, visited) => {
+    const steps = next(id).filter(step => !visited.has(step.to))
+    if (!steps.length) {
+      if (path.length >= 3 || path.some(step => step.edge && step.edge.type === 'calls-api')) chains.push(path)
+      return
+    }
+    steps.forEach(step => walk(step.to, [...path, step], new Set([...visited, step.to])))
+  }
+  results
+    .slice(0, CHAIN_FILES)
+    .filter(result => !hasIncoming(result.id) && !TEST_FILE.test(result.path))
+    .forEach(result => walk(result.id, [{ to: result.id, edge: null }], new Set([result.id])))
+
+  const covered = new Set()
+  return chains
+    .sort((a, b) => b.length - a.length)
+    .filter(chain => {
+      const fresh = chain.some(step => !covered.has(step.to))
+      chain.forEach(step => covered.add(step.to))
+      return fresh
+    })
+    .slice(0, 3)
+    .map(chain =>
+      chain.map(step => ({
+        repo: graph.files[step.to].repo,
+        path: graph.files[step.to].path,
+        via: step.edge ? step.edge.type : null,
+        detail: step.edge && step.edge.type === 'calls-api' ? step.edge.detail : null,
+      })),
+    )
+}
+
+const confidenceFor = (known, unknown, lex, results, graph, top) => {
+  const reasons = []
+  if (!known.length) {
+    return { level: 'low', reasons: ['none of the task words appear in the code'] }
+  }
+  const best = results.length ? lex.get(results[0].id) : null
+  const coverage = best ? best.covered : 0
+  const topFive = new Set(results.slice(0, 5).map(result => result.id))
+  const connected = results
+    .slice(0, 5)
+    .some(result => graph.out[result.id].some(edge => topFive.has(edge.to)))
+  const scores = results.map(result => result.score)
+  const clear = scores.length > 5 ? scores[0] > scores[5] * 1.3 : true
+
+  if (unknown.length) reasons.push(`words not found in the code: ${unknown.join(', ')}`)
+  reasons.push(`best file matches ${Math.round(coverage * 100)}% of the task words`)
+  reasons.push(connected ? 'top files are connected to each other' : 'top files are not connected to each other')
+  if (!clear) reasons.push('many files score about the same')
+
+  let level = 'low'
+  if (coverage >= 0.7 && connected && !unknown.length && clear) level = 'high'
+  else if (coverage >= 0.5 && (connected || clear)) level = 'medium'
+  return { level, reasons }
+}
+
+const KIND_FILE = /^(.+?)\.([a-z][a-z0-9-]*)\.([a-z]+)$/
+
+const placementFor = (graph, results) => {
+  const byRepo = new Map()
+  graph.files.forEach(file => {
+    if (!byRepo.has(file.repo)) byRepo.set(file.repo, [])
+    byRepo.get(file.repo).push(file.path)
+  })
+  const patterns = new Map()
+  results.slice(0, 8).forEach(result => {
+    const dir = Path.dirname(result.path)
+    const base = Path.basename(result.path)
+    const match = base.match(KIND_FILE)
+    const paths = byRepo.get(result.repo)
+    let pattern = null
+    let count = 0
+    if (match && Path.basename(dir) === match[1]) {
+      const parent = Path.dirname(dir)
+      pattern = `${parent}/<name>/<name>.${match[2]}.${match[3]}`
+      const re = new RegExp(`^${escape(parent)}/([^/]+)/\\1\\.${escape(match[2])}\\.${match[3]}$`)
+      count = paths.filter(path => re.test(path)).length
+    } else if (match) {
+      pattern = `${dir}/<name>.${match[2]}.${match[3]}`
+      const re = new RegExp(`^${escape(dir)}/[^/]+\\.${escape(match[2])}\\.${match[3]}$`)
+      count = paths.filter(path => re.test(path)).length
+    }
+    if (pattern && count >= 3 && !patterns.has(`${result.repo}\0${pattern}`)) {
+      patterns.set(`${result.repo}\0${pattern}`, { repo: result.repo, pattern, count })
+    }
+  })
+  return [...patterns.values()].slice(0, 5)
+}
+
+const escape = value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+
+module.exports = {
+  createSearch,
+}

package/src/find/text.js

@@ -0,0 +1,86 @@
+'use strict'
+
+const STOPWORDS = new Set(`
+a an the and or but if then else when while of to in on at by for from with without into onto over under
+is are was were be been being am do does did done has have had having can could should would will shall may might must
+it its this that these those there here what which who whom whose where why how all any each every some such no not only
+own same so than too very just also as i me my we our us you your he she they them their his her
+please need needs want wants like get gets let lets use using used via per etc e g ie eg
+`.split(/\s+/).filter(Boolean))
+
+const stem = word => {
+  let w = word
+  if (w.length > 4 && w.endsWith('ies')) w = `${w.slice(0, -3)}y`
+  else if (w.length > 4 && /(ss|x|z|ch|sh)es$/.test(w)) w = w.slice(0, -2)
+  else if (w.length > 3 && w.endsWith('s') && !/(ss|us|is)$/.test(w)) w = w.slice(0, -1)
+  if ((w.length > 5 && w.endsWith('ing')) || (w.length > 4 && w.endsWith('ed'))) {
+    w = w.slice(0, w.endsWith('ing') ? -3 : -2)
+    if (w.length > 4 && /([^aeiouls])\1$/.test(w)) w = w.slice(0, -1)
+  }
+  if (w.length > 3 && w.endsWith('e')) w = w.slice(0, -1)
+  return w
+}
+
+const words = value =>
+  String(value || '')
+    .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+    .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter(word => word && !/^\d+$/.test(word))
+
+const splitTerms = value => {
+  const parts = words(value).filter(word => !STOPWORDS.has(word))
+  const single = parts.filter(word => word.length > 1).map(stem)
+  const original = new Map(parts.filter(word => word.length > 1).map(word => [stem(word), word]))
+  const joined = []
+  for (let i = 0; i < parts.length - 1; i++) {
+    if (parts[i].length > 1 && parts[i + 1].length > 1) joined.push(stem(parts[i] + parts[i + 1]))
+  }
+  String(value || '')
+    .split(/[^A-Za-z0-9_]+/)
+    .forEach(token => {
+      const pieces = words(token)
+      if (pieces.length > 2) joined.push(stem(pieces.join('')))
+    })
+  return { single, joined, original }
+}
+
+const identifier = value => {
+  const pieces = words(value)
+  return pieces.length > 1 ? stem(pieces.join('')) : null
+}
+
+const terms = value => {
+  const { single, joined } = splitTerms(value)
+  return [...single, ...joined]
+}
+
+const editDistance = (a, b, max) => {
+  if (Math.abs(a.length - b.length) > max) return max + 1
+  let previous = Array.from({ length: b.length + 1 }, (_, i) => i)
+  for (let i = 1; i <= a.length; i++) {
+    const current = [i]
+    let best = i
+    for (let j = 1; j <= b.length; j++) {
+      current[j] = Math.min(
+        previous[j] + 1,
+        current[j - 1] + 1,
+        previous[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1),
+      )
+      best = Math.min(best, current[j])
+    }
+    if (best > max) return max + 1
+    previous = current
+  }
+  return previous[b.length]
+}
+
+module.exports = {
+  stem,
+  words,
+  terms,
+  splitTerms,
+  identifier,
+  editDistance,
+}

package/src/git-helper.js

@@ -0,0 +1,102 @@
+'use strict'
+
+const Crypto = require('crypto')
+const Fs = require('fs')
+const Path = require('path')
+const { execFileSync } = require('child_process')
+
+const git = (dir, args) => {
+  try {
+    return execFileSync('git', ['-C', dir, ...args], {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    }).trim()
+  } catch (err) {
+    return null
+  }
+}
+
+const isRepo = dir => Fs.existsSync(Path.join(dir, '.git'))
+
+const findRepos = folder => {
+  if (isRepo(folder)) {
+    return [folder]
+  }
+  return Fs.readdirSync(folder, { withFileTypes: true })
+    .filter(entry => entry.isDirectory() && !entry.name.startsWith('.'))
+    .map(entry => Path.join(folder, entry.name))
+    .filter(isRepo)
+    .sort()
+}
+
+const remoteUrl = dir => git(dir, ['remote', 'get-url', 'origin'])
+
+const defaultBranch = dir => {
+  const head = git(dir, ['symbolic-ref', '--short', 'refs/remotes/origin/HEAD'])
+  if (head) {
+    return head.replace(/^origin\//, '')
+  }
+  const main = ['master', 'main'].find(name => git(dir, ['rev-parse', '--verify', '--quiet', `refs/remotes/origin/${name}`]))
+  if (main) {
+    return main
+  }
+  const current = git(dir, ['rev-parse', '--abbrev-ref', 'HEAD'])
+  return current && current !== 'HEAD' ? current : null
+}
+
+const FETCH_TIMEOUT = 60 * 1000
+
+const fetchBranch = (dir, branch) => {
+  if (!branch) return false
+  try {
+    execFileSync('git', ['-C', dir, 'fetch', '--quiet', '--no-tags', 'origin', branch], {
+      stdio: 'ignore',
+      timeout: FETCH_TIMEOUT,
+      env: { ...process.env, GIT_TERMINAL_PROMPT: '0', GIT_SSH_COMMAND: process.env.GIT_SSH_COMMAND || 'ssh -o BatchMode=yes' },
+    })
+    return true
+  } catch (err) {
+    return false
+  }
+}
+
+const normalizeGitUrl = url =>
+  String(url)
+    .trim()
+    .replace(/^[a-z+]+:\/\//i, '')
+    .replace(/^[^@/]+@/, '')
+    .replace(/:(?!\d+\/)/, '/')
+    .replace(/:\d+\//, '/')
+    .replace(/\/+$/, '')
+    .replace(/\.git$/i, '')
+    .toLowerCase()
+
+const firstCommit = dir => {
+  if (git(dir, ['rev-parse', '--is-shallow-repository']) !== 'false') {
+    return null
+  }
+  const roots = git(dir, ['rev-list', '--max-parents=0', 'HEAD'])
+  return roots ? roots.split('\n').sort()[0] : null
+}
+
+const identify = dir => {
+  const url = remoteUrl(dir)
+  return {
+    fingerprint: firstCommit(dir),
+    remoteHash: url
+      ? Crypto.createHash('sha256').update(normalizeGitUrl(url)).digest('hex')
+      : null,
+  }
+}
+
+const hideCredentials = url => String(url).replace(/(https?:\/\/)[^@/]+@/i, '$1')
+
+module.exports = {
+  findRepos,
+  remoteUrl,
+  defaultBranch,
+  fetchBranch,
+  normalizeGitUrl,
+  identify,
+  hideCredentials,
+}