solidity-argus 0.1.8 → 0.3.0

package/src/skills/analysis/cluster.ts

@@ -0,0 +1,414 @@
+import { tokenJaccard } from "./similarity"
+
+/** Input finding from the PDF extraction pipeline */
+export interface ClusterFinding {
+  title: string
+  severity: string
+  description: string
+  category: string
+  source_pdf: string
+  source_name?: string
+}
+
+/** A single cluster of related findings */
+export interface FindingCluster {
+  id: number
+  category: string
+  members: ClusterFinding[]
+  medoid: ClusterFinding
+  medoidIndex: number
+  topTokens: string[]
+  avgInternalSimilarity: number
+  size: number
+}
+
+/** Configuration for clustering */
+export interface ClusterConfig {
+  linkThreshold: number
+  cohesionMinSimilarity: number
+  minClusterSize: number
+}
+
+/** Full clustering result */
+export interface ClusterResult {
+  clusters: FindingCluster[]
+  singletons: ClusterFinding[]
+  stats: {
+    totalFindings: number
+    totalClusters: number
+    totalSingletons: number
+    categoryCounts: Record<string, number>
+    largestCluster: number
+    avgClusterSize: number
+  }
+}
+
+export const DEFAULT_CLUSTER_CONFIG: ClusterConfig = {
+  linkThreshold: 0.6,
+  cohesionMinSimilarity: 0.65,
+  minClusterSize: 2,
+}
+
+const STOPWORDS = new Set([
+  "the",
+  "a",
+  "an",
+  "is",
+  "are",
+  "was",
+  "were",
+  "be",
+  "been",
+  "being",
+  "have",
+  "has",
+  "had",
+  "do",
+  "does",
+  "did",
+  "will",
+  "would",
+  "shall",
+  "should",
+  "may",
+  "might",
+  "can",
+  "could",
+  "of",
+  "in",
+  "to",
+  "for",
+  "with",
+  "on",
+  "at",
+  "by",
+  "from",
+  "as",
+  "into",
+  "through",
+  "during",
+  "before",
+  "after",
+  "above",
+  "below",
+  "between",
+  "out",
+  "off",
+  "over",
+  "under",
+  "again",
+  "further",
+  "then",
+  "once",
+  "here",
+  "there",
+  "where",
+  "when",
+  "how",
+  "all",
+  "each",
+  "every",
+  "both",
+  "few",
+  "more",
+  "most",
+  "other",
+  "some",
+  "such",
+  "no",
+  "nor",
+  "not",
+  "only",
+  "own",
+  "same",
+  "than",
+  "too",
+  "very",
+  "and",
+  "but",
+  "or",
+  "if",
+  "this",
+  "that",
+  "these",
+  "those",
+  "it",
+  "its",
+  "contract",
+  "function",
+  "solidity",
+  "smart",
+  "vulnerability",
+  "attack",
+  "attacker",
+  "token",
+  "address",
+  "value",
+  "state",
+  "require",
+  "modifier",
+  "external",
+  "internal",
+  "public",
+  "private",
+  "mapping",
+  "uint256",
+  "bool",
+  "returns",
+  "event",
+  "emit",
+])
+
+class UnionFind {
+  parent: number[]
+  rank: number[]
+  constructor(n: number) {
+    this.parent = Array.from({ length: n }, (_, index) => index)
+    this.rank = Array.from({ length: n }, () => 0)
+  }
+  find(x: number): number {
+    const parent = this.parent[x] ?? x
+    if (parent === x) return x
+    const root = this.find(parent)
+    this.parent[x] = root
+    return root
+  }
+  union(x: number, y: number): boolean {
+    const rootX = this.find(x)
+    const rootY = this.find(y)
+    if (rootX === rootY) return false
+    const rankX = this.rank[rootX] ?? 0
+    const rankY = this.rank[rootY] ?? 0
+    if (rankX < rankY) {
+      this.parent[rootX] = rootY
+      return true
+    }
+
+    if (rankX > rankY) {
+      this.parent[rootY] = rootX
+      return true
+    }
+
+    this.parent[rootY] = rootX
+    this.rank[rootX] = rankX + 1
+    return true
+  }
+  components(): Map<number, number[]> {
+    const groups = new Map<number, number[]>()
+    for (let index = 0; index < this.parent.length; index += 1) {
+      const root = this.find(index)
+      const group = groups.get(root) ?? []
+      group.push(index)
+      groups.set(root, group)
+    }
+    return groups
+  }
+}
+
+function tokenize(text: string): string[] {
+  if (!text) return []
+  const deduped = new Set<string>()
+  for (const token of text.toLowerCase().split(/[^a-z0-9]+/g)) {
+    if (token.length < 3) continue
+    if (STOPWORDS.has(token)) continue
+    deduped.add(token)
+  }
+  return Array.from(deduped)
+}
+
+function computeTokenSets(findings: ClusterFinding[]): Set<string>[] {
+  return findings.map((finding) => new Set(tokenize(`${finding.title} ${finding.description}`)))
+}
+
+function buildSimilarityMatrix(tokenSets: Set<string>[]): number[][] {
+  const tokenArrays = tokenSets.map((set) => [...set])
+  const matrix: number[][] = Array.from({ length: tokenSets.length }, () =>
+    Array.from({ length: tokenSets.length }, () => 0),
+  )
+  for (let i = 0; i < tokenSets.length; i += 1) {
+    const rowI = matrix[i]
+    if (rowI) rowI[i] = 1
+    for (let j = i + 1; j < tokenSets.length; j += 1) {
+      const similarity = tokenJaccard(tokenArrays[i] ?? [], tokenArrays[j] ?? [])
+      if (rowI) rowI[j] = similarity
+      const rowJ = matrix[j]
+      if (rowJ) rowJ[i] = similarity
+    }
+  }
+  return matrix
+}
+
+function medoidForMembers(memberIndices: number[], similarityMatrix: number[][]): number {
+  if (memberIndices.length === 0) return -1
+  if (memberIndices.length === 1) return memberIndices[0] ?? -1
+  let bestIndex = memberIndices[0] ?? -1
+  let bestAvgSimilarity = -1
+  for (const candidateIndex of memberIndices) {
+    let total = 0
+    for (const otherIndex of memberIndices) {
+      if (candidateIndex === otherIndex) continue
+      total += similarityMatrix[candidateIndex]?.[otherIndex] ?? 0
+    }
+
+    const avg = total / (memberIndices.length - 1)
+    if (avg > bestAvgSimilarity) {
+      bestAvgSimilarity = avg
+      bestIndex = candidateIndex
+    }
+  }
+  return bestIndex
+}
+
+function averageInternalSimilarity(memberIndices: number[], similarityMatrix: number[][]): number {
+  if (memberIndices.length < 2) return 0
+  let total = 0
+  let pairs = 0
+  for (let i = 0; i < memberIndices.length; i += 1) {
+    const left = memberIndices[i] ?? -1
+
+    for (let j = i + 1; j < memberIndices.length; j += 1) {
+      const right = memberIndices[j] ?? -1
+      total += similarityMatrix[left]?.[right] ?? 0
+      pairs += 1
+    }
+  }
+  return total / pairs
+}
+
+function topTokensForMembers(memberIndices: number[], tokenSets: Set<string>[]): string[] {
+  const counts = new Map<string, number>()
+  for (const memberIndex of memberIndices) {
+    const tokenSet = tokenSets[memberIndex]
+    if (!tokenSet) continue
+    for (const token of tokenSet) {
+      counts.set(token, (counts.get(token) ?? 0) + 1)
+    }
+  }
+  return Array.from(counts.entries())
+    .sort((left, right) => {
+      const countDelta = right[1] - left[1]
+      if (countDelta !== 0) return countDelta
+      return left[0].localeCompare(right[0])
+    })
+    .slice(0, 10)
+    .map(([token]) => token)
+}
+
+function pushSingletons(
+  target: ClusterFinding[],
+  bucket: ClusterFinding[],
+  indices: number[],
+): void {
+  for (const index of indices) {
+    const finding = bucket[index]
+    if (finding) target.push(finding)
+  }
+}
+
+/**
+ * Groups related findings per category using token Jaccard links and
+ * union-find connected components, then peels low-cohesion outliers.
+ */
+export function clusterFindings(
+  findings: ClusterFinding[],
+  config: ClusterConfig = DEFAULT_CLUSTER_CONFIG,
+): ClusterResult {
+  const clusters: FindingCluster[] = []
+  const singletons: ClusterFinding[] = []
+  const categoryCounts: Record<string, number> = {}
+  const buckets = new Map<string, ClusterFinding[]>()
+  for (const finding of findings) {
+    categoryCounts[finding.category] = (categoryCounts[finding.category] ?? 0) + 1
+    const bucket = buckets.get(finding.category)
+    if (bucket) {
+      bucket.push(finding)
+    } else {
+      buckets.set(finding.category, [finding])
+    }
+  }
+  const orderedCategories = Array.from(buckets.keys()).sort((left, right) =>
+    left.localeCompare(right),
+  )
+  let nextClusterId = 1
+  for (const category of orderedCategories) {
+    const bucket = buckets.get(category) ?? []
+    if (bucket.length < config.minClusterSize) {
+      singletons.push(...bucket)
+      continue
+    }
+    const tokenSets = computeTokenSets(bucket)
+    const similarityMatrix = buildSimilarityMatrix(tokenSets)
+    const uf = new UnionFind(bucket.length)
+    for (let i = 0; i < bucket.length; i += 1) {
+      for (let j = i + 1; j < bucket.length; j += 1) {
+        if ((similarityMatrix[i]?.[j] ?? 0) < config.linkThreshold) continue
+        uf.union(i, j)
+      }
+    }
+    for (const memberIndices of uf.components().values()) {
+      if (memberIndices.length < config.minClusterSize) {
+        pushSingletons(singletons, bucket, memberIndices)
+        continue
+      }
+      const initialMedoid = medoidForMembers(memberIndices, similarityMatrix)
+      const keptIndices: number[] = []
+      const peeledIndices: number[] = []
+      for (const index of memberIndices) {
+        if (index === initialMedoid) {
+          keptIndices.push(index)
+          continue
+        }
+        const similarityToMedoid = similarityMatrix[initialMedoid]?.[index] ?? 0
+        if (similarityToMedoid >= config.cohesionMinSimilarity) {
+          keptIndices.push(index)
+        } else {
+          peeledIndices.push(index)
+        }
+      }
+      pushSingletons(singletons, bucket, peeledIndices)
+      if (keptIndices.length < config.minClusterSize) {
+        pushSingletons(singletons, bucket, keptIndices)
+        continue
+      }
+      const finalMedoid = medoidForMembers(keptIndices, similarityMatrix)
+      const members = keptIndices
+        .map((index) => bucket[index])
+        .filter((finding): finding is ClusterFinding => Boolean(finding))
+      if (members.length < config.minClusterSize) {
+        singletons.push(...members)
+        continue
+      }
+      const medoidIndex = Math.max(0, keptIndices.indexOf(finalMedoid))
+      const medoid = members.at(medoidIndex)
+      if (!medoid) throw new Error("Medoid index out of bounds — this should not happen")
+      clusters.push({
+        id: nextClusterId,
+        category,
+        members,
+        medoid,
+        medoidIndex,
+        topTokens: topTokensForMembers(keptIndices, tokenSets),
+        avgInternalSimilarity: averageInternalSimilarity(keptIndices, similarityMatrix),
+        size: members.length,
+      })
+      nextClusterId += 1
+    }
+  }
+  const largestCluster = clusters.reduce((max, cluster) => Math.max(max, cluster.size), 0)
+  const avgClusterSize =
+    clusters.length === 0
+      ? 0
+      : clusters.reduce((total, cluster) => total + cluster.size, 0) / clusters.length
+  return {
+    clusters,
+    singletons,
+    stats: {
+      totalFindings: findings.length,
+      totalClusters: clusters.length,
+      totalSingletons: singletons.length,
+      categoryCounts,
+      largestCluster,
+      avgClusterSize,
+    },
+  }
+}

package/src/skills/analysis/gates.ts

@@ -0,0 +1,227 @@
+import type { SkillDoc } from "./normalize"
+import type { SimilarityPair, SimilarityScore } from "./similarity"
+
+export type GateLevel = "block" | "warn" | "info" | "pass"
+
+export interface GateVerdict {
+  level: GateLevel
+  reason: string
+}
+
+export interface GateConfig {
+  blockThreshold: number
+  warnThreshold: number
+  infoThreshold: number
+  blockExactRegexConflict: boolean
+}
+
+export interface SkillReport {
+  totalSkills: number
+  findings: Array<{
+    skillA: string
+    skillB: string
+    score: SimilarityScore
+    verdict: GateVerdict
+  }>
+  summary: { block: number; warn: number; info: number }
+}
+
+export const DEFAULT_GATE_CONFIG: GateConfig = {
+  blockThreshold: 0.9,
+  warnThreshold: 0.78,
+  infoThreshold: 0.65,
+  blockExactRegexConflict: true,
+}
+
+const LEVEL_ORDER: Record<GateLevel, number> = {
+  block: 0,
+  warn: 1,
+  info: 2,
+  pass: 3,
+}
+
+function formatScore(score: number): string {
+  return score.toFixed(2)
+}
+
+function normalizeRegex(rule: string): string {
+  return rule.replace(/\s+/g, " ").trim()
+}
+
+function pairKey(skillA: string, skillB: string): string {
+  return skillA < skillB ? `${skillA}|||${skillB}` : `${skillB}|||${skillA}`
+}
+
+function topSignals(score: SimilarityScore): string {
+  const signals = [
+    { label: "body TF-IDF", value: score.bodyTfidf },
+    { label: "body shingles", value: score.bodyShingle },
+    { label: "name/description", value: score.nameDesc },
+    { label: "detection rules", value: score.detectionRules },
+  ]
+
+  signals.sort((left, right) => right.value - left.value)
+  return signals
+    .slice(0, 2)
+    .map((signal) => `${signal.label} ${formatScore(signal.value)}`)
+    .join(", ")
+}
+
+function scoreForConflict(score: SimilarityScore | undefined): SimilarityScore {
+  if (score) return score
+
+  return {
+    composite: 1,
+    bodyTfidf: 0,
+    bodyShingle: 0,
+    nameDesc: 0,
+    detectionRules: 1,
+  }
+}
+
+export function evaluatePair(
+  pair: SimilarityPair,
+  config: GateConfig = DEFAULT_GATE_CONFIG,
+): GateVerdict {
+  const composite = pair.score.composite
+  const signalSummary = topSignals(pair.score)
+  const reasonSuffix = `composite ${formatScore(composite)}; top signals: ${signalSummary}`
+
+  if (composite >= config.blockThreshold) {
+    return { level: "block", reason: `Duplicate risk: ${reasonSuffix}` }
+  }
+
+  if (composite >= config.warnThreshold) {
+    return { level: "warn", reason: `Near-duplicate risk: ${reasonSuffix}` }
+  }
+
+  if (composite >= config.infoThreshold) {
+    return { level: "info", reason: `Related skills: ${reasonSuffix}` }
+  }
+
+  return { level: "pass", reason: `Below thresholds: ${reasonSuffix}` }
+}
+
+export function checkExactRegexConflicts(
+  docs: SkillDoc[],
+): Array<{ skillA: string; skillB: string; sharedRegex: string }> {
+  const conflicts: Array<{ skillA: string; skillB: string; sharedRegex: string }> = []
+
+  for (let i = 0; i < docs.length; i += 1) {
+    const docA = docs[i]
+    if (!docA) continue
+
+    const rulesA = new Set(
+      docA.detectionRules.map(normalizeRegex).filter((rule) => rule.length > 0),
+    )
+
+    for (let j = i + 1; j < docs.length; j += 1) {
+      const docB = docs[j]
+      if (!docB) continue
+      if (docA.name === docB.name) continue
+
+      const rulesB = new Set(
+        docB.detectionRules.map(normalizeRegex).filter((rule) => rule.length > 0),
+      )
+      for (const sharedRegex of rulesA) {
+        if (!rulesB.has(sharedRegex)) continue
+        conflicts.push({
+          skillA: docA.name,
+          skillB: docB.name,
+          sharedRegex,
+        })
+      }
+    }
+  }
+
+  return conflicts
+}
+
+export function generateReport(
+  docs: SkillDoc[],
+  pairs: SimilarityPair[],
+  config: GateConfig = DEFAULT_GATE_CONFIG,
+): SkillReport {
+  const findings: SkillReport["findings"] = []
+  const pairScores = new Map<string, SimilarityScore>()
+
+  for (const pair of pairs) {
+    pairScores.set(pairKey(pair.skillA, pair.skillB), pair.score)
+
+    const verdict = evaluatePair(pair, config)
+    if (verdict.level === "pass") continue
+
+    findings.push({
+      skillA: pair.skillA,
+      skillB: pair.skillB,
+      score: pair.score,
+      verdict,
+    })
+  }
+
+  if (config.blockExactRegexConflict) {
+    const indexByPairKey = new Map<string, number>()
+    findings.forEach((finding, index) => {
+      indexByPairKey.set(pairKey(finding.skillA, finding.skillB), index)
+    })
+
+    const conflicts = checkExactRegexConflicts(docs)
+    for (const conflict of conflicts) {
+      const key = pairKey(conflict.skillA, conflict.skillB)
+      const existingIndex = indexByPairKey.get(key)
+      const conflictReason = `Exact detection rule conflict: ${conflict.sharedRegex}`
+
+      if (existingIndex !== undefined) {
+        const existing = findings[existingIndex]
+        if (!existing) continue
+        existing.verdict = { level: "block", reason: conflictReason }
+        continue
+      }
+
+      findings.push({
+        skillA: conflict.skillA,
+        skillB: conflict.skillB,
+        score: scoreForConflict(pairScores.get(key)),
+        verdict: { level: "block", reason: conflictReason },
+      })
+      indexByPairKey.set(key, findings.length - 1)
+    }
+  }
+
+  findings.sort((left, right) => {
+    const levelDelta = LEVEL_ORDER[left.verdict.level] - LEVEL_ORDER[right.verdict.level]
+    if (levelDelta !== 0) return levelDelta
+    return right.score.composite - left.score.composite
+  })
+
+  const summary = { block: 0, warn: 0, info: 0 }
+  for (const finding of findings) {
+    if (finding.verdict.level === "block") summary.block += 1
+    if (finding.verdict.level === "warn") summary.warn += 1
+    if (finding.verdict.level === "info") summary.info += 1
+  }
+
+  return {
+    totalSkills: docs.length,
+    findings,
+    summary,
+  }
+}
+
+export function formatReportText(report: SkillReport): string {
+  const lines = [
+    `Skills: ${report.totalSkills} | Blocks: ${report.summary.block} | Warnings: ${report.summary.warn} | Info: ${report.summary.info}`,
+  ]
+
+  for (const finding of report.findings) {
+    lines.push(
+      `[${finding.verdict.level.toUpperCase()}] ${finding.skillA} ↔ ${finding.skillB} (${formatScore(finding.score.composite)}) — ${finding.verdict.reason}`,
+    )
+  }
+
+  return lines.join("\n")
+}
+
+export function formatReportJson(report: SkillReport): string {
+  return JSON.stringify(report, null, 2)
+}

package/src/skills/analysis/index.ts

@@ -0,0 +1,33 @@
+export {
+  type ClusterConfig,
+  type ClusterFinding,
+  type ClusterResult,
+  clusterFindings,
+  DEFAULT_CLUSTER_CONFIG,
+  type FindingCluster,
+} from "./cluster"
+export {
+  checkExactRegexConflicts,
+  DEFAULT_GATE_CONFIG,
+  evaluatePair,
+  formatReportJson,
+  formatReportText,
+  type GateConfig,
+  type GateLevel,
+  type GateVerdict,
+  generateReport,
+  type SkillReport,
+} from "./gates"
+export { normalizeSkill, type SkillDoc } from "./normalize"
+export {
+  buildTfidfCorpus,
+  computeAllPairs,
+  computeSimilarity,
+  detectionRuleOverlap,
+  type SimilarityPair,
+  type SimilarityScore,
+  shingleJaccard,
+  type TfidfCorpus,
+  tfidfCosine,
+  tokenJaccard,
+} from "./similarity"