solidity-argus 0.1.8 → 0.3.0

package/src/skills/analysis/normalize.ts

@@ -0,0 +1,217 @@
+import { parseFrontmatter } from "../skill-schema"
+
+export interface SkillDoc {
+  name: string
+  description: string
+  category: string | undefined
+  detectionRules: string[]
+  bodyText: string
+  bodyTokens: string[]
+  nameDescTokens: string[]
+  ruleTokens: string[]
+}
+
+const STOPWORDS = new Set([
+  "the",
+  "a",
+  "an",
+  "is",
+  "are",
+  "was",
+  "were",
+  "be",
+  "been",
+  "being",
+  "have",
+  "has",
+  "had",
+  "do",
+  "does",
+  "did",
+  "will",
+  "would",
+  "shall",
+  "should",
+  "may",
+  "might",
+  "can",
+  "could",
+  "of",
+  "in",
+  "to",
+  "for",
+  "with",
+  "on",
+  "at",
+  "by",
+  "from",
+  "as",
+  "into",
+  "through",
+  "during",
+  "before",
+  "after",
+  "above",
+  "below",
+  "between",
+  "out",
+  "off",
+  "over",
+  "under",
+  "again",
+  "further",
+  "then",
+  "once",
+  "here",
+  "there",
+  "where",
+  "when",
+  "how",
+  "all",
+  "each",
+  "every",
+  "both",
+  "few",
+  "more",
+  "most",
+  "other",
+  "some",
+  "such",
+  "no",
+  "nor",
+  "not",
+  "only",
+  "own",
+  "same",
+  "than",
+  "too",
+  "very",
+  "and",
+  "but",
+  "or",
+  "if",
+  "this",
+  "that",
+  "these",
+  "those",
+  "it",
+  "its",
+  "contract",
+  "function",
+  "solidity",
+  "smart",
+  "vulnerability",
+  "attack",
+  "attacker",
+  "token",
+  "address",
+  "value",
+  "state",
+  "require",
+  "modifier",
+  "external",
+  "internal",
+  "public",
+  "private",
+  "mapping",
+  "uint256",
+  "bool",
+  "returns",
+  "event",
+  "emit",
+])
+
+function stripFrontmatter(content: string): string {
+  return content.replace(/^---[ \t]*\r?\n[\s\S]*?\r?\n---[ \t]*\r?\n?/, "")
+}
+
+function stripCodeBlocks(content: string): string {
+  return content.replace(/```[\s\S]*?```/g, " ")
+}
+
+function stripHtmlComments(content: string): string {
+  return content.replace(/<!--[\s\S]*?-->/g, " ")
+}
+
+function normalizeWhitespace(content: string): string {
+  return content.toLowerCase().replace(/\s+/g, " ").trim()
+}
+
+function tokenize(text: string): string[] {
+  if (!text) return []
+
+  return text
+    .toLowerCase()
+    .split(/[^a-z0-9]+/g)
+    .filter((token) => token.length >= 3)
+    .filter((token) => !STOPWORDS.has(token))
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null
+}
+
+function extractDetectionRules(frontmatter: Record<string, unknown>): string[] {
+  const rawRules = frontmatter.detection_rules
+  if (!Array.isArray(rawRules)) return []
+
+  const rules: string[] = []
+  for (const rule of rawRules) {
+    if (!isRecord(rule)) continue
+    if (typeof rule.regex !== "string") continue
+    rules.push(rule.regex)
+  }
+
+  return rules
+}
+
+function normalizeRuleToken(token: string): string {
+  return token.replace(/^[_.]+|[_.]+$/g, "").toLowerCase()
+}
+
+function extractRuleTokens(rules: string[]): string[] {
+  const tokens: string[] = []
+
+  for (const rule of rules) {
+    const parts = rule.split(/[^a-zA-Z0-9_.]+/g)
+    for (const part of parts) {
+      const normalized = normalizeRuleToken(part)
+      if (!normalized) continue
+      if (normalized.length < 3) continue
+      tokens.push(normalized)
+    }
+  }
+
+  return tokens
+}
+
+export function normalizeSkill(content: string): SkillDoc | null {
+  if (!content.trim()) return null
+
+  const frontmatter = parseFrontmatter(content)
+  if (!frontmatter) return null
+
+  const rawName = frontmatter.name
+  if (typeof rawName !== "string" || !rawName.trim()) return null
+
+  const name = rawName.trim()
+  const description = typeof frontmatter.description === "string" ? frontmatter.description : ""
+  const category = typeof frontmatter.category === "string" ? frontmatter.category : undefined
+
+  const detectionRules = extractDetectionRules(frontmatter)
+  const bodyWithoutFrontmatter = stripFrontmatter(content)
+  const withoutComments = stripHtmlComments(bodyWithoutFrontmatter)
+  const withoutCode = stripCodeBlocks(withoutComments)
+  const bodyText = normalizeWhitespace(withoutCode)
+
+  return {
+    name,
+    description,
+    category,
+    detectionRules,
+    bodyText,
+    bodyTokens: tokenize(bodyText),
+    nameDescTokens: tokenize(`${name} ${description}`),
+    ruleTokens: extractRuleTokens(detectionRules),
+  }
+}

package/src/skills/analysis/similarity.ts

@@ -0,0 +1,224 @@
+import type { SkillDoc } from "./normalize"
+
+export interface SimilarityScore {
+  composite: number
+  bodyTfidf: number
+  bodyShingle: number
+  nameDesc: number
+  detectionRules: number
+}
+
+export interface SimilarityPair {
+  skillA: string
+  skillB: string
+  score: SimilarityScore
+}
+
+export interface TfidfCorpus {
+  docCount: number
+  docFreq: Map<string, number>
+}
+
+const BODY_TFIDF_WEIGHT = 0.45
+const BODY_SHINGLE_WEIGHT = 0.2
+const NAME_DESC_WEIGHT = 0.2
+const DETECTION_RULES_WEIGHT = 0.15
+
+function clamp01(value: number): number {
+  if (!Number.isFinite(value)) return 0
+  if (value < 0) return 0
+  if (value > 1) return 1
+  return value
+}
+
+function getTokenCounts(tokens: string[]): Map<string, number> {
+  const counts = new Map<string, number>()
+  for (const token of tokens) {
+    counts.set(token, (counts.get(token) ?? 0) + 1)
+  }
+  return counts
+}
+
+function buildTfIdfVector(doc: SkillDoc, corpus: TfidfCorpus): Map<string, number> {
+  const vector = new Map<string, number>()
+  const totalTokens = doc.bodyTokens.length
+  const docCount = corpus.docCount
+
+  if (totalTokens === 0 || docCount === 0) {
+    return vector
+  }
+
+  const tokenCounts = getTokenCounts(doc.bodyTokens)
+
+  for (const [token, count] of tokenCounts) {
+    const df = corpus.docFreq.get(token)
+    if (!df || df <= 0) continue
+
+    const tf = count / totalTokens
+    const idf = Math.log(docCount / df)
+    const weight = tf * idf
+    if (weight === 0) continue
+
+    vector.set(token, weight)
+  }
+
+  return vector
+}
+
+function dotProduct(a: Map<string, number>, b: Map<string, number>): number {
+  if (a.size === 0 || b.size === 0) return 0
+
+  let dot = 0
+  const [small, large] = a.size < b.size ? [a, b] : [b, a]
+  for (const [token, weight] of small) {
+    dot += weight * (large.get(token) ?? 0)
+  }
+
+  return dot
+}
+
+function vectorNorm(vector: Map<string, number>): number {
+  let sumSquares = 0
+  for (const weight of vector.values()) {
+    sumSquares += weight * weight
+  }
+  return Math.sqrt(sumSquares)
+}
+
+function buildShingleSet(tokens: string[], n: number): Set<string> {
+  const shingles = new Set<string>()
+  if (tokens.length < n || n <= 0) return shingles
+
+  for (let i = 0; i <= tokens.length - n; i += 1) {
+    shingles.add(tokens.slice(i, i + n).join(" "))
+  }
+
+  return shingles
+}
+
+function setIntersectionSize<T>(a: Set<T>, b: Set<T>): number {
+  if (a.size === 0 || b.size === 0) return 0
+
+  let count = 0
+  const [small, large] = a.size < b.size ? [a, b] : [b, a]
+  for (const value of small) {
+    if (large.has(value)) count += 1
+  }
+  return count
+}
+
+function normalizeRegex(rule: string): string {
+  return rule.replace(/\s+/g, " ").trim()
+}
+
+export function buildTfidfCorpus(docs: SkillDoc[]): TfidfCorpus {
+  const docFreq = new Map<string, number>()
+
+  for (const doc of docs) {
+    const uniqueTokens = new Set(doc.bodyTokens)
+    for (const token of uniqueTokens) {
+      docFreq.set(token, (docFreq.get(token) ?? 0) + 1)
+    }
+  }
+
+  return {
+    docCount: docs.length,
+    docFreq,
+  }
+}
+
+export function tfidfCosine(a: SkillDoc, b: SkillDoc, corpus: TfidfCorpus): number {
+  const vectorA = buildTfIdfVector(a, corpus)
+  const vectorB = buildTfIdfVector(b, corpus)
+  if (vectorA.size === 0 || vectorB.size === 0) return 0
+
+  const normA = vectorNorm(vectorA)
+  const normB = vectorNorm(vectorB)
+  if (normA === 0 || normB === 0) return 0
+
+  const similarity = dotProduct(vectorA, vectorB) / (normA * normB)
+  return clamp01(similarity)
+}
+
+export function shingleJaccard(a: string[], b: string[], n: number = 4): number {
+  const setA = buildShingleSet(a, n)
+  const setB = buildShingleSet(b, n)
+  if (setA.size === 0 && setB.size === 0) return 0
+
+  const intersection = setIntersectionSize(setA, setB)
+  const union = setA.size + setB.size - intersection
+  if (union === 0) return 0
+
+  return clamp01(intersection / union)
+}
+
+export function tokenJaccard(a: string[], b: string[]): number {
+  const setA = new Set(a)
+  const setB = new Set(b)
+  if (setA.size === 0 && setB.size === 0) return 0
+
+  const intersection = setIntersectionSize(setA, setB)
+  const union = setA.size + setB.size - intersection
+  if (union === 0) return 0
+
+  return clamp01(intersection / union)
+}
+
+export function detectionRuleOverlap(a: SkillDoc, b: SkillDoc): number {
+  const normalizedA = a.detectionRules.map(normalizeRegex)
+  const normalizedB = b.detectionRules.map(normalizeRegex)
+  const setA = new Set(normalizedA)
+  const setB = new Set(normalizedB)
+
+  const maxRuleCount = Math.max(normalizedA.length, normalizedB.length)
+  const sharedExact = setIntersectionSize(setA, setB)
+  const exactMatch = maxRuleCount === 0 ? 0 : sharedExact / maxRuleCount
+  const tokenOverlap = tokenJaccard(a.ruleTokens, b.ruleTokens)
+
+  return clamp01(exactMatch * 0.6 + tokenOverlap * 0.4)
+}
+
+export function computeSimilarity(a: SkillDoc, b: SkillDoc, corpus: TfidfCorpus): SimilarityScore {
+  const bodyTfidf = clamp01(tfidfCosine(a, b, corpus))
+  const bodyShingle = clamp01(shingleJaccard(a.bodyTokens, b.bodyTokens, 4))
+  const nameDesc = clamp01(tokenJaccard(a.nameDescTokens, b.nameDescTokens))
+  const detectionRules = clamp01(detectionRuleOverlap(a, b))
+
+  const composite = clamp01(
+    bodyTfidf * BODY_TFIDF_WEIGHT +
+      bodyShingle * BODY_SHINGLE_WEIGHT +
+      nameDesc * NAME_DESC_WEIGHT +
+      detectionRules * DETECTION_RULES_WEIGHT,
+  )
+
+  return {
+    composite,
+    bodyTfidf,
+    bodyShingle,
+    nameDesc,
+    detectionRules,
+  }
+}
+
+export function computeAllPairs(docs: SkillDoc[], corpus: TfidfCorpus): SimilarityPair[] {
+  const pairs: SimilarityPair[] = []
+
+  for (let i = 0; i < docs.length; i += 1) {
+    const skillA = docs[i]
+    if (!skillA) continue
+
+    for (let j = i + 1; j < docs.length; j += 1) {
+      const skillB = docs[j]
+      if (!skillB) continue
+
+      pairs.push({
+        skillA: skillA.name,
+        skillB: skillB.name,
+        score: computeSimilarity(skillA, skillB, corpus),
+      })
+    }
+  }
+
+  pairs.sort((left, right) => right.score.composite - left.score.composite)
+  return pairs
+}

package/src/skills/argus-skill-resolver.ts

@@ -0,0 +1,237 @@
+import { type Dirent, existsSync, readdirSync, readFileSync } from "node:fs"
+import { homedir } from "node:os"
+import { basename, extname, join, resolve } from "node:path"
+import type { ArgusConfig } from "../config/types"
+import { createLogger } from "../shared/logger"
+import { parseFrontmatter, validateSkillFrontmatter } from "./skill-schema"
+
+export type ResolvedSkill = {
+  name: string
+  description: string
+  filePath: string
+  source: "bundled" | "custom" | "trailofbits" | "opencode" | "claude"
+  content: string
+  source_url?: string
+  source_license?: string
+  imported_at?: string
+  source_hash?: string
+}
+
+const OMO_PROJECT_SKILLS_DIR = [".opencode", "skills"]
+const OMO_GLOBAL_SKILLS_DIR = [".config", "opencode", "skills"]
+const CLAUDE_PROJECT_SKILLS_DIR = [".claude", "skills"]
+const CLAUDE_GLOBAL_SKILLS_DIR = [".claude", "skills"]
+const TOB_CACHE_DIR = join(homedir(), ".cache", "solidity-argus", "trailofbits-skills")
+
+const SKILL_NAME_ALIASES: Record<string, string> = {
+  "vulnerability-patterns/reentrancy": "reentrancy",
+  "vulnerability-patterns/oracle-manipulation": "oracle-manipulation",
+  "vulnerability-patterns/access-control": "access-control",
+  "protocol-patterns/amm-dex": "amm-dex",
+  "protocol-patterns/lending-borrowing": "lending-borrowing",
+  "checklists/cyfrin-best-practices-upgrades": "cyfrin-best-practices-upgrades",
+  "references/exploit-reference": "exploit-reference",
+  "building-secure-contracts/token-integration-analyzer": "token-integration-analyzer",
+}
+
+function inferSkillNameFromPath(filePath: string): string {
+  if (basename(filePath) === "SKILL.md") {
+    return basename(resolve(filePath, ".."))
+  }
+  return basename(filePath, extname(filePath))
+}
+
+function parseSkillNameFromFrontmatter(content: string): string | null {
+  const match = content.match(/^name:\s*(.+)$/m)
+  if (!match) return null
+  return match[1]?.trim().replace(/^"|"$/g, "") ?? null
+}
+
+function parseSkillDescriptionFromFrontmatter(content: string): string {
+  const match = content.match(/^description:\s*(.+)$/m)
+  if (!match) return ""
+  const raw = match[1]?.trim() ?? ""
+  if (raw === ">" || raw === ">-") return ""
+  return raw.replace(/^"|"$/g, "")
+}
+
+function collectMarkdownFiles(root: string, maxDepth = 8): string[] {
+  if (!existsSync(root)) return []
+
+  const files: string[] = []
+  const stack: Array<{ dir: string; depth: number }> = [{ dir: root, depth: 0 }]
+
+  while (stack.length > 0) {
+    const current = stack.pop()
+    if (!current) continue
+    const { dir, depth } = current
+
+    let entries: Dirent[]
+    try {
+      entries = readdirSync(dir, { withFileTypes: true })
+    } catch {
+      continue
+    }
+
+    for (const entry of entries) {
+      const fullPath = join(dir, entry.name)
+      if (entry.isDirectory()) {
+        if (depth < maxDepth) stack.push({ dir: fullPath, depth: depth + 1 })
+        continue
+      }
+
+      if (!entry.isFile()) continue
+      if (extname(entry.name).toLowerCase() !== ".md") continue
+      files.push(fullPath)
+    }
+  }
+
+  return files
+}
+
+function getTrailOfBitsRoots(): string[] {
+  const pluginsDir = join(TOB_CACHE_DIR, "plugins")
+  if (!existsSync(pluginsDir)) return []
+
+  let entries: Dirent[]
+  try {
+    entries = readdirSync(pluginsDir, { withFileTypes: true })
+  } catch {
+    return []
+  }
+
+  const roots: string[] = []
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue
+    const skillsDir = join(pluginsDir, entry.name, "skills")
+    if (existsSync(skillsDir)) roots.push(skillsDir)
+  }
+  return roots
+}
+
+export function normalizeSkillName(input: string): string {
+  const trimmed = input.trim()
+  const alias = SKILL_NAME_ALIASES[trimmed]
+  if (alias) return alias
+  if (trimmed.includes("/")) {
+    const last = trimmed.split("/").at(-1)
+    if (last) return last
+  }
+  return trimmed
+}
+
+type SkillRoot = {
+  path: string
+  source: ResolvedSkill["source"]
+}
+
+function resolveCustomSkillsRoot(projectDir: string, argusConfig?: ArgusConfig): string | null {
+  const customSkillsDir = argusConfig?.knowledge?.customSkillsDir
+  if (!customSkillsDir) return null
+  const resolvedCustom = customSkillsDir.startsWith("/")
+    ? customSkillsDir
+    : resolve(projectDir, customSkillsDir)
+  return existsSync(resolvedCustom) ? resolvedCustom : null
+}
+
+export function resolveSkillRoots(projectDir: string, argusConfig?: ArgusConfig): SkillRoot[] {
+  const precedence = argusConfig?.knowledge?.skillPrecedence ?? "bundled-first"
+
+  const bundledRoot: SkillRoot = {
+    path: resolve(import.meta.dir, "../../skills"),
+    source: "bundled",
+  }
+  const customRoot = resolveCustomSkillsRoot(projectDir, argusConfig)
+  const customSkillRoot: SkillRoot | null = customRoot
+    ? { path: customRoot, source: "custom" }
+    : null
+
+  const roots: SkillRoot[] = []
+
+  if (precedence === "custom-first") {
+    if (customSkillRoot) roots.push(customSkillRoot)
+    roots.push(bundledRoot)
+  } else {
+    roots.push(bundledRoot)
+    if (customSkillRoot) roots.push(customSkillRoot)
+  }
+
+  for (const tobRoot of getTrailOfBitsRoots()) {
+    roots.push({ path: tobRoot, source: "trailofbits" })
+  }
+
+  roots.push({ path: join(projectDir, ...OMO_PROJECT_SKILLS_DIR), source: "opencode" })
+  roots.push({ path: join(homedir(), ...OMO_GLOBAL_SKILLS_DIR), source: "opencode" })
+  roots.push({ path: join(projectDir, ...CLAUDE_PROJECT_SKILLS_DIR), source: "claude" })
+  roots.push({ path: join(homedir(), ...CLAUDE_GLOBAL_SKILLS_DIR), source: "claude" })
+
+  const seen = new Set<string>()
+  return roots.filter((root) => {
+    if (!existsSync(root.path)) return false
+    if (seen.has(root.path)) return false
+    seen.add(root.path)
+    return true
+  })
+}
+
+export function resolveArgusSkills(
+  projectDir: string,
+  argusConfig?: ArgusConfig,
+): Map<string, ResolvedSkill> {
+  const resolved = new Map<string, ResolvedSkill>()
+  const roots = resolveSkillRoots(projectDir, argusConfig)
+  const logger = createLogger()
+
+  for (const root of roots) {
+    const markdownFiles = collectMarkdownFiles(root.path)
+    for (const markdownFile of markdownFiles) {
+      let content: string
+      try {
+        content = readFileSync(markdownFile, "utf8")
+      } catch {
+        continue
+      }
+
+      const frontmatter = parseFrontmatter(content)
+      if (frontmatter) {
+        const validation = validateSkillFrontmatter(frontmatter)
+        if (!validation.success) {
+          logger.warn(
+            `Skipping skill with invalid frontmatter: ${markdownFile} — ${validation.errors.join(", ")}`,
+          )
+          continue
+        }
+      }
+
+      const parsedName = parseSkillNameFromFrontmatter(content)
+      const rawName = parsedName || inferSkillNameFromPath(markdownFile)
+      const normalizedName = normalizeSkillName(rawName)
+      if (!normalizedName) continue
+      if (resolved.has(normalizedName)) continue
+
+      const skill: ResolvedSkill = {
+        name: normalizedName,
+        description: parseSkillDescriptionFromFrontmatter(content),
+        filePath: markdownFile,
+        source: root.source,
+        content,
+      }
+
+      if (frontmatter) {
+        if (typeof frontmatter.source_url === "string") skill.source_url = frontmatter.source_url
+        if (typeof frontmatter.source_license === "string")
+          skill.source_license = frontmatter.source_license
+        if (typeof frontmatter.imported_at === "string") skill.imported_at = frontmatter.imported_at
+        if (typeof frontmatter.source_hash === "string") skill.source_hash = frontmatter.source_hash
+      }
+
+      resolved.set(normalizedName, skill)
+    }
+  }
+
+  return resolved
+}
+
+export function getRequiredAuditSkills(): string[] {
+  return ["reentrancy", "oracle-manipulation", "amm-dex"]
+}