solidity-argus 0.2.0 → 0.3.2

package/src/skills/analysis/normalize.ts

@@ -0,0 +1,217 @@
+import { parseFrontmatter } from "../skill-schema"
+
+export interface SkillDoc {
+  name: string
+  description: string
+  category: string | undefined
+  detectionRules: string[]
+  bodyText: string
+  bodyTokens: string[]
+  nameDescTokens: string[]
+  ruleTokens: string[]
+}
+
+const STOPWORDS = new Set([
+  "the",
+  "a",
+  "an",
+  "is",
+  "are",
+  "was",
+  "were",
+  "be",
+  "been",
+  "being",
+  "have",
+  "has",
+  "had",
+  "do",
+  "does",
+  "did",
+  "will",
+  "would",
+  "shall",
+  "should",
+  "may",
+  "might",
+  "can",
+  "could",
+  "of",
+  "in",
+  "to",
+  "for",
+  "with",
+  "on",
+  "at",
+  "by",
+  "from",
+  "as",
+  "into",
+  "through",
+  "during",
+  "before",
+  "after",
+  "above",
+  "below",
+  "between",
+  "out",
+  "off",
+  "over",
+  "under",
+  "again",
+  "further",
+  "then",
+  "once",
+  "here",
+  "there",
+  "where",
+  "when",
+  "how",
+  "all",
+  "each",
+  "every",
+  "both",
+  "few",
+  "more",
+  "most",
+  "other",
+  "some",
+  "such",
+  "no",
+  "nor",
+  "not",
+  "only",
+  "own",
+  "same",
+  "than",
+  "too",
+  "very",
+  "and",
+  "but",
+  "or",
+  "if",
+  "this",
+  "that",
+  "these",
+  "those",
+  "it",
+  "its",
+  "contract",
+  "function",
+  "solidity",
+  "smart",
+  "vulnerability",
+  "attack",
+  "attacker",
+  "token",
+  "address",
+  "value",
+  "state",
+  "require",
+  "modifier",
+  "external",
+  "internal",
+  "public",
+  "private",
+  "mapping",
+  "uint256",
+  "bool",
+  "returns",
+  "event",
+  "emit",
+])
+
+function stripFrontmatter(content: string): string {
+  return content.replace(/^---[ \t]*\r?\n[\s\S]*?\r?\n---[ \t]*\r?\n?/, "")
+}
+
+function stripCodeBlocks(content: string): string {
+  return content.replace(/```[\s\S]*?```/g, " ")
+}
+
+function stripHtmlComments(content: string): string {
+  return content.replace(/<!--[\s\S]*?-->/g, " ")
+}
+
+function normalizeWhitespace(content: string): string {
+  return content.toLowerCase().replace(/\s+/g, " ").trim()
+}
+
+function tokenize(text: string): string[] {
+  if (!text) return []
+
+  return text
+    .toLowerCase()
+    .split(/[^a-z0-9]+/g)
+    .filter((token) => token.length >= 3)
+    .filter((token) => !STOPWORDS.has(token))
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null
+}
+
+function extractDetectionRules(frontmatter: Record<string, unknown>): string[] {
+  const rawRules = frontmatter.detection_rules
+  if (!Array.isArray(rawRules)) return []
+
+  const rules: string[] = []
+  for (const rule of rawRules) {
+    if (!isRecord(rule)) continue
+    if (typeof rule.regex !== "string") continue
+    rules.push(rule.regex)
+  }
+
+  return rules
+}
+
+function normalizeRuleToken(token: string): string {
+  return token.replace(/^[_.]+|[_.]+$/g, "").toLowerCase()
+}
+
+function extractRuleTokens(rules: string[]): string[] {
+  const tokens: string[] = []
+
+  for (const rule of rules) {
+    const parts = rule.split(/[^a-zA-Z0-9_.]+/g)
+    for (const part of parts) {
+      const normalized = normalizeRuleToken(part)
+      if (!normalized) continue
+      if (normalized.length < 3) continue
+      tokens.push(normalized)
+    }
+  }
+
+  return tokens
+}
+
+export function normalizeSkill(content: string): SkillDoc | null {
+  if (!content.trim()) return null
+
+  const frontmatter = parseFrontmatter(content)
+  if (!frontmatter) return null
+
+  const rawName = frontmatter.name
+  if (typeof rawName !== "string" || !rawName.trim()) return null
+
+  const name = rawName.trim()
+  const description = typeof frontmatter.description === "string" ? frontmatter.description : ""
+  const category = typeof frontmatter.category === "string" ? frontmatter.category : undefined
+
+  const detectionRules = extractDetectionRules(frontmatter)
+  const bodyWithoutFrontmatter = stripFrontmatter(content)
+  const withoutComments = stripHtmlComments(bodyWithoutFrontmatter)
+  const withoutCode = stripCodeBlocks(withoutComments)
+  const bodyText = normalizeWhitespace(withoutCode)
+
+  return {
+    name,
+    description,
+    category,
+    detectionRules,
+    bodyText,
+    bodyTokens: tokenize(bodyText),
+    nameDescTokens: tokenize(`${name} ${description}`),
+    ruleTokens: extractRuleTokens(detectionRules),
+  }
+}

package/src/skills/analysis/similarity.ts

@@ -0,0 +1,224 @@
+import type { SkillDoc } from "./normalize"
+
+export interface SimilarityScore {
+  composite: number
+  bodyTfidf: number
+  bodyShingle: number
+  nameDesc: number
+  detectionRules: number
+}
+
+export interface SimilarityPair {
+  skillA: string
+  skillB: string
+  score: SimilarityScore
+}
+
+export interface TfidfCorpus {
+  docCount: number
+  docFreq: Map<string, number>
+}
+
+const BODY_TFIDF_WEIGHT = 0.45
+const BODY_SHINGLE_WEIGHT = 0.2
+const NAME_DESC_WEIGHT = 0.2
+const DETECTION_RULES_WEIGHT = 0.15
+
+function clamp01(value: number): number {
+  if (!Number.isFinite(value)) return 0
+  if (value < 0) return 0
+  if (value > 1) return 1
+  return value
+}
+
+function getTokenCounts(tokens: string[]): Map<string, number> {
+  const counts = new Map<string, number>()
+  for (const token of tokens) {
+    counts.set(token, (counts.get(token) ?? 0) + 1)
+  }
+  return counts
+}
+
+function buildTfIdfVector(doc: SkillDoc, corpus: TfidfCorpus): Map<string, number> {
+  const vector = new Map<string, number>()
+  const totalTokens = doc.bodyTokens.length
+  const docCount = corpus.docCount
+
+  if (totalTokens === 0 || docCount === 0) {
+    return vector
+  }
+
+  const tokenCounts = getTokenCounts(doc.bodyTokens)
+
+  for (const [token, count] of tokenCounts) {
+    const df = corpus.docFreq.get(token)
+    if (!df || df <= 0) continue
+
+    const tf = count / totalTokens
+    const idf = Math.log(docCount / df)
+    const weight = tf * idf
+    if (weight === 0) continue
+
+    vector.set(token, weight)
+  }
+
+  return vector
+}
+
+function dotProduct(a: Map<string, number>, b: Map<string, number>): number {
+  if (a.size === 0 || b.size === 0) return 0
+
+  let dot = 0
+  const [small, large] = a.size < b.size ? [a, b] : [b, a]
+  for (const [token, weight] of small) {
+    dot += weight * (large.get(token) ?? 0)
+  }
+
+  return dot
+}
+
+function vectorNorm(vector: Map<string, number>): number {
+  let sumSquares = 0
+  for (const weight of vector.values()) {
+    sumSquares += weight * weight
+  }
+  return Math.sqrt(sumSquares)
+}
+
+function buildShingleSet(tokens: string[], n: number): Set<string> {
+  const shingles = new Set<string>()
+  if (tokens.length < n || n <= 0) return shingles
+
+  for (let i = 0; i <= tokens.length - n; i += 1) {
+    shingles.add(tokens.slice(i, i + n).join(" "))
+  }
+
+  return shingles
+}
+
+function setIntersectionSize<T>(a: Set<T>, b: Set<T>): number {
+  if (a.size === 0 || b.size === 0) return 0
+
+  let count = 0
+  const [small, large] = a.size < b.size ? [a, b] : [b, a]
+  for (const value of small) {
+    if (large.has(value)) count += 1
+  }
+  return count
+}
+
+function normalizeRegex(rule: string): string {
+  return rule.replace(/\s+/g, " ").trim()
+}
+
+export function buildTfidfCorpus(docs: SkillDoc[]): TfidfCorpus {
+  const docFreq = new Map<string, number>()
+
+  for (const doc of docs) {
+    const uniqueTokens = new Set(doc.bodyTokens)
+    for (const token of uniqueTokens) {
+      docFreq.set(token, (docFreq.get(token) ?? 0) + 1)
+    }
+  }
+
+  return {
+    docCount: docs.length,
+    docFreq,
+  }
+}
+
+export function tfidfCosine(a: SkillDoc, b: SkillDoc, corpus: TfidfCorpus): number {
+  const vectorA = buildTfIdfVector(a, corpus)
+  const vectorB = buildTfIdfVector(b, corpus)
+  if (vectorA.size === 0 || vectorB.size === 0) return 0
+
+  const normA = vectorNorm(vectorA)
+  const normB = vectorNorm(vectorB)
+  if (normA === 0 || normB === 0) return 0
+
+  const similarity = dotProduct(vectorA, vectorB) / (normA * normB)
+  return clamp01(similarity)
+}
+
+export function shingleJaccard(a: string[], b: string[], n: number = 4): number {
+  const setA = buildShingleSet(a, n)
+  const setB = buildShingleSet(b, n)
+  if (setA.size === 0 && setB.size === 0) return 0
+
+  const intersection = setIntersectionSize(setA, setB)
+  const union = setA.size + setB.size - intersection
+  if (union === 0) return 0
+
+  return clamp01(intersection / union)
+}
+
+export function tokenJaccard(a: string[], b: string[]): number {
+  const setA = new Set(a)
+  const setB = new Set(b)
+  if (setA.size === 0 && setB.size === 0) return 0
+
+  const intersection = setIntersectionSize(setA, setB)
+  const union = setA.size + setB.size - intersection
+  if (union === 0) return 0
+
+  return clamp01(intersection / union)
+}
+
+export function detectionRuleOverlap(a: SkillDoc, b: SkillDoc): number {
+  const normalizedA = a.detectionRules.map(normalizeRegex)
+  const normalizedB = b.detectionRules.map(normalizeRegex)
+  const setA = new Set(normalizedA)
+  const setB = new Set(normalizedB)
+
+  const maxRuleCount = Math.max(normalizedA.length, normalizedB.length)
+  const sharedExact = setIntersectionSize(setA, setB)
+  const exactMatch = maxRuleCount === 0 ? 0 : sharedExact / maxRuleCount
+  const tokenOverlap = tokenJaccard(a.ruleTokens, b.ruleTokens)
+
+  return clamp01(exactMatch * 0.6 + tokenOverlap * 0.4)
+}
+
+export function computeSimilarity(a: SkillDoc, b: SkillDoc, corpus: TfidfCorpus): SimilarityScore {
+  const bodyTfidf = clamp01(tfidfCosine(a, b, corpus))
+  const bodyShingle = clamp01(shingleJaccard(a.bodyTokens, b.bodyTokens, 4))
+  const nameDesc = clamp01(tokenJaccard(a.nameDescTokens, b.nameDescTokens))
+  const detectionRules = clamp01(detectionRuleOverlap(a, b))
+
+  const composite = clamp01(
+    bodyTfidf * BODY_TFIDF_WEIGHT +
+      bodyShingle * BODY_SHINGLE_WEIGHT +
+      nameDesc * NAME_DESC_WEIGHT +
+      detectionRules * DETECTION_RULES_WEIGHT,
+  )
+
+  return {
+    composite,
+    bodyTfidf,
+    bodyShingle,
+    nameDesc,
+    detectionRules,
+  }
+}
+
+export function computeAllPairs(docs: SkillDoc[], corpus: TfidfCorpus): SimilarityPair[] {
+  const pairs: SimilarityPair[] = []
+
+  for (let i = 0; i < docs.length; i += 1) {
+    const skillA = docs[i]
+    if (!skillA) continue
+
+    for (let j = i + 1; j < docs.length; j += 1) {
+      const skillB = docs[j]
+      if (!skillB) continue
+
+      pairs.push({
+        skillA: skillA.name,
+        skillB: skillB.name,
+        score: computeSimilarity(skillA, skillB, corpus),
+      })
+    }
+  }
+
+  pairs.sort((left, right) => right.score.composite - left.score.composite)
+  return pairs
+}

package/src/skills/argus-skill-resolver.ts

@@ -1,4 +1,4 @@
-import { existsSync, readdirSync, readFileSync, type Dirent } from "node:fs"
+import { type Dirent, existsSync, readdirSync, readFileSync } from "node:fs"
 import { homedir } from "node:os"
 import { basename, extname, join, resolve } from "node:path"
 import type { ArgusConfig } from "../config/types"
@@ -137,9 +137,14 @@ function resolveCustomSkillsRoot(projectDir: string, argusConfig?: ArgusConfig):
 export function resolveSkillRoots(projectDir: string, argusConfig?: ArgusConfig): SkillRoot[] {
   const precedence = argusConfig?.knowledge?.skillPrecedence ?? "bundled-first"
 
-  const bundledRoot: SkillRoot = { path: resolve(import.meta.dir, "../../skills"), source: "bundled" }
+  const bundledRoot: SkillRoot = {
+    path: resolve(import.meta.dir, "../../skills"),
+    source: "bundled",
+  }
   const customRoot = resolveCustomSkillsRoot(projectDir, argusConfig)
-  const customSkillRoot: SkillRoot | null = customRoot ? { path: customRoot, source: "custom" } : null
+  const customSkillRoot: SkillRoot | null = customRoot
+    ? { path: customRoot, source: "custom" }
+    : null
 
   const roots: SkillRoot[] = []
 
@@ -169,7 +174,10 @@ export function resolveSkillRoots(projectDir: string, argusConfig?: ArgusConfig)
   })
 }
 
-export function resolveArgusSkills(projectDir: string, argusConfig?: ArgusConfig): Map<string, ResolvedSkill> {
+export function resolveArgusSkills(
+  projectDir: string,
+  argusConfig?: ArgusConfig,
+): Map<string, ResolvedSkill> {
   const resolved = new Map<string, ResolvedSkill>()
   const roots = resolveSkillRoots(projectDir, argusConfig)
   const logger = createLogger()
@@ -188,7 +196,9 @@ export function resolveArgusSkills(projectDir: string, argusConfig?: ArgusConfig
       if (frontmatter) {
         const validation = validateSkillFrontmatter(frontmatter)
         if (!validation.success) {
-          logger.warn(`Skipping skill with invalid frontmatter: ${markdownFile} — ${validation.errors.join(", ")}`)
+          logger.warn(
+            `Skipping skill with invalid frontmatter: ${markdownFile} — ${validation.errors.join(", ")}`,
+          )
           continue
         }
       }
@@ -209,7 +219,8 @@ export function resolveArgusSkills(projectDir: string, argusConfig?: ArgusConfig
 
       if (frontmatter) {
         if (typeof frontmatter.source_url === "string") skill.source_url = frontmatter.source_url
-        if (typeof frontmatter.source_license === "string") skill.source_license = frontmatter.source_license
+        if (typeof frontmatter.source_license === "string")
+          skill.source_license = frontmatter.source_license
         if (typeof frontmatter.imported_at === "string") skill.imported_at = frontmatter.imported_at
         if (typeof frontmatter.source_hash === "string") skill.source_hash = frontmatter.source_hash
       }

package/src/skills/skill-schema.ts

@@ -1,5 +1,9 @@
-import { z } from "zod"
 import { parse as parseYaml } from "yaml"
+import { z } from "zod"
+import { createLogger } from "../shared/logger"
+import { PATTERN_CATEGORIES } from "../tools/pattern-schema"
+
+const logger = createLogger()
 
 export const DetectionRuleSchema = z.object({
   regex: z.string(),
@@ -23,19 +27,14 @@ export const SkillFrontmatterSchema = z.object({
   deprecated: z.boolean().optional(),
   replacement: z.string().optional(),
   category: z
-    .enum([
-      "vulnerability-pattern",
-      "methodology",
-      "protocol-pattern",
-      "checklist",
-      "reference",
-    ])
+    .enum(["vulnerability-pattern", "methodology", "protocol-pattern", "checklist", "reference"])
     .optional(),
   source_url: z.string().url().optional(),
   source_license: z.string().optional(),
   imported_at: z.string().optional(),
   source_hash: z.string().optional(),
   detection_rules: z.array(DetectionRuleSchema).optional(),
+  pattern_category: z.enum(PATTERN_CATEGORIES).optional(),
 })
 
 export type SkillFrontmatter = z.infer<typeof SkillFrontmatterSchema>
@@ -68,7 +67,9 @@ export function parseFrontmatter(content: string): Record<string, unknown> | nul
       if (typeof parsed === "object" && parsed !== null) {
         return parsed as Record<string, unknown>
       }
-    } catch {}
+    } catch {
+      logger.debug("YAML frontmatter parse failed, falling back to line parser")
+    }
   }
 
   const lines = raw.split(/\r?\n/)
@@ -78,7 +79,7 @@ export function parseFrontmatter(content: string): Record<string, unknown> | nul
     const kvMatch = line.match(/^([\w][\w-]*):\s*(.*)$/)
     if (!kvMatch) continue
 
-    const key = kvMatch[1]!
+    const key = kvMatch[1] ?? ""
     let raw = kvMatch[2]?.trim() ?? ""
 
     if ((raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'"))) {