webmcp-cli 1.0.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/features/agent-simulator.d.ts +67 -0
- package/dist/agent/features/agent-simulator.js +368 -0
- package/dist/agent/features/agent-simulator.js.map +1 -0
- package/dist/agent/features/index.d.ts +8 -0
- package/dist/agent/features/index.js +9 -0
- package/dist/agent/features/index.js.map +1 -0
- package/dist/agent/features/simulation-judge.d.ts +78 -0
- package/dist/agent/features/simulation-judge.js +276 -0
- package/dist/agent/features/simulation-judge.js.map +1 -0
- package/dist/agent/features/test-case-generator.d.ts +35 -0
- package/dist/agent/features/test-case-generator.js +257 -0
- package/dist/agent/features/test-case-generator.js.map +1 -0
- package/dist/agent/index.d.ts +7 -0
- package/dist/agent/index.js +10 -0
- package/dist/agent/index.js.map +1 -0
- package/dist/agent/llm-client.d.ts +76 -0
- package/dist/agent/llm-client.js +198 -0
- package/dist/agent/llm-client.js.map +1 -0
- package/dist/audit/run-single-page-audit.d.ts +41 -0
- package/dist/audit/run-single-page-audit.js +103 -0
- package/dist/audit/run-single-page-audit.js.map +1 -0
- package/dist/bin/webmcp.d.ts +5 -0
- package/dist/bin/webmcp.js +14 -0
- package/dist/bin/webmcp.js.map +1 -0
- package/dist/browser/audit-runner.d.ts +30 -0
- package/dist/browser/audit-runner.js +77 -0
- package/dist/browser/audit-runner.js.map +1 -0
- package/dist/browser/index.d.ts +6 -0
- package/dist/browser/index.js +7 -0
- package/dist/browser/index.js.map +1 -0
- package/dist/browser/interceptor.d.ts +68 -0
- package/dist/browser/interceptor.js +257 -0
- package/dist/browser/interceptor.js.map +1 -0
- package/dist/browser/playwright.d.ts +98 -0
- package/dist/browser/playwright.js +158 -0
- package/dist/browser/playwright.js.map +1 -0
- package/dist/cli/commands/audit.d.ts +12 -0
- package/dist/cli/commands/audit.js +349 -0
- package/dist/cli/commands/audit.js.map +1 -0
- package/dist/cli/commands/interactive.d.ts +10 -0
- package/dist/cli/commands/interactive.js +34 -0
- package/dist/cli/commands/interactive.js.map +1 -0
- package/dist/cli/index.d.ts +17 -0
- package/dist/cli/index.js +84 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/options/parse-audit-options.d.ts +12 -0
- package/dist/cli/options/parse-audit-options.js +64 -0
- package/dist/cli/options/parse-audit-options.js.map +1 -0
- package/dist/core/constants.d.ts +102 -0
- package/dist/core/constants.js +214 -0
- package/dist/core/constants.js.map +1 -0
- package/dist/core/types/audit.d.ts +260 -0
- package/dist/core/types/audit.js +5 -0
- package/dist/core/types/audit.js.map +1 -0
- package/dist/core/types/index.d.ts +6 -0
- package/dist/core/types/index.js +7 -0
- package/dist/core/types/index.js.map +1 -0
- package/dist/core/types/rule.d.ts +190 -0
- package/dist/core/types/rule.js +26 -0
- package/dist/core/types/rule.js.map +1 -0
- package/dist/core/types/tool.d.ts +312 -0
- package/dist/core/types/tool.js +6 -0
- package/dist/core/types/tool.js.map +1 -0
- package/dist/detection/declarative.d.ts +27 -0
- package/dist/detection/declarative.js +343 -0
- package/dist/detection/declarative.js.map +1 -0
- package/dist/detection/imperative.d.ts +38 -0
- package/dist/detection/imperative.js +99 -0
- package/dist/detection/imperative.js.map +1 -0
- package/dist/detection/index.d.ts +5 -0
- package/dist/detection/index.js +6 -0
- package/dist/detection/index.js.map +1 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.js +19 -0
- package/dist/index.js.map +1 -0
- package/dist/llm/advice-service.d.ts +38 -0
- package/dist/llm/advice-service.js +243 -0
- package/dist/llm/advice-service.js.map +1 -0
- package/dist/llm/evaluator.d.ts +89 -0
- package/dist/llm/evaluator.js +274 -0
- package/dist/llm/evaluator.js.map +1 -0
- package/dist/llm/index.d.ts +11 -0
- package/dist/llm/index.js +15 -0
- package/dist/llm/index.js.map +1 -0
- package/dist/llm/json-response.d.ts +12 -0
- package/dist/llm/json-response.js +67 -0
- package/dist/llm/json-response.js.map +1 -0
- package/dist/llm/providers/mock.d.ts +29 -0
- package/dist/llm/providers/mock.js +324 -0
- package/dist/llm/providers/mock.js.map +1 -0
- package/dist/llm/providers/openrouter.d.ts +53 -0
- package/dist/llm/providers/openrouter.js +321 -0
- package/dist/llm/providers/openrouter.js.map +1 -0
- package/dist/llm/request-cache.d.ts +28 -0
- package/dist/llm/request-cache.js +99 -0
- package/dist/llm/request-cache.js.map +1 -0
- package/dist/llm/types.d.ts +233 -0
- package/dist/llm/types.js +7 -0
- package/dist/llm/types.js.map +1 -0
- package/dist/rules/best-practices/BP-001.d.ts +11 -0
- package/dist/rules/best-practices/BP-001.js +56 -0
- package/dist/rules/best-practices/BP-001.js.map +1 -0
- package/dist/rules/best-practices/BP-002.d.ts +11 -0
- package/dist/rules/best-practices/BP-002.js +63 -0
- package/dist/rules/best-practices/BP-002.js.map +1 -0
- package/dist/rules/best-practices/BP-003.d.ts +11 -0
- package/dist/rules/best-practices/BP-003.js +68 -0
- package/dist/rules/best-practices/BP-003.js.map +1 -0
- package/dist/rules/coverage/COV-001.d.ts +8 -0
- package/dist/rules/coverage/COV-001.js +51 -0
- package/dist/rules/coverage/COV-001.js.map +1 -0
- package/dist/rules/description/DESC-003.d.ts +13 -0
- package/dist/rules/description/DESC-003.js +96 -0
- package/dist/rules/description/DESC-003.js.map +1 -0
- package/dist/rules/description/DESC-004.d.ts +8 -0
- package/dist/rules/description/DESC-004.js +61 -0
- package/dist/rules/description/DESC-004.js.map +1 -0
- package/dist/rules/description/DESC-005.d.ts +12 -0
- package/dist/rules/description/DESC-005.js +70 -0
- package/dist/rules/description/DESC-005.js.map +1 -0
- package/dist/rules/description/index.d.ts +4 -0
- package/dist/rules/description/index.js +5 -0
- package/dist/rules/description/index.js.map +1 -0
- package/dist/rules/implementation/IMP-001.d.ts +10 -0
- package/dist/rules/implementation/IMP-001.js +36 -0
- package/dist/rules/implementation/IMP-001.js.map +1 -0
- package/dist/rules/implementation/IMP-003.d.ts +9 -0
- package/dist/rules/implementation/IMP-003.js +45 -0
- package/dist/rules/implementation/IMP-003.js.map +1 -0
- package/dist/rules/implementation/IMP-004.d.ts +9 -0
- package/dist/rules/implementation/IMP-004.js +48 -0
- package/dist/rules/implementation/IMP-004.js.map +1 -0
- package/dist/rules/implementation/IMP-005.d.ts +9 -0
- package/dist/rules/implementation/IMP-005.js +54 -0
- package/dist/rules/implementation/IMP-005.js.map +1 -0
- package/dist/rules/implementation/IMP-007.d.ts +8 -0
- package/dist/rules/implementation/IMP-007.js +79 -0
- package/dist/rules/implementation/IMP-007.js.map +1 -0
- package/dist/rules/implementation/IMP-013.d.ts +9 -0
- package/dist/rules/implementation/IMP-013.js +55 -0
- package/dist/rules/implementation/IMP-013.js.map +1 -0
- package/dist/rules/implementation/index.d.ts +9 -0
- package/dist/rules/implementation/index.js +10 -0
- package/dist/rules/implementation/index.js.map +1 -0
- package/dist/rules/index.d.ts +51 -0
- package/dist/rules/index.js +100 -0
- package/dist/rules/index.js.map +1 -0
- package/dist/rules/llm/LLM-001.d.ts +14 -0
- package/dist/rules/llm/LLM-001.js +78 -0
- package/dist/rules/llm/LLM-001.js.map +1 -0
- package/dist/rules/llm/LLM-002.d.ts +14 -0
- package/dist/rules/llm/LLM-002.js +77 -0
- package/dist/rules/llm/LLM-002.js.map +1 -0
- package/dist/rules/llm/LLM-003.d.ts +16 -0
- package/dist/rules/llm/LLM-003.js +82 -0
- package/dist/rules/llm/LLM-003.js.map +1 -0
- package/dist/rules/llm/LLM-004.d.ts +14 -0
- package/dist/rules/llm/LLM-004.js +87 -0
- package/dist/rules/llm/LLM-004.js.map +1 -0
- package/dist/rules/llm/LLM-005.d.ts +16 -0
- package/dist/rules/llm/LLM-005.js +105 -0
- package/dist/rules/llm/LLM-005.js.map +1 -0
- package/dist/rules/llm/index.d.ts +10 -0
- package/dist/rules/llm/index.js +11 -0
- package/dist/rules/llm/index.js.map +1 -0
- package/dist/rules/runner.d.ts +54 -0
- package/dist/rules/runner.js +138 -0
- package/dist/rules/runner.js.map +1 -0
- package/dist/rules/schema/SCHEMA-001.d.ts +9 -0
- package/dist/rules/schema/SCHEMA-001.js +57 -0
- package/dist/rules/schema/SCHEMA-001.js.map +1 -0
- package/dist/rules/schema/SCHEMA-002.d.ts +9 -0
- package/dist/rules/schema/SCHEMA-002.js +59 -0
- package/dist/rules/schema/SCHEMA-002.js.map +1 -0
- package/dist/rules/schema/SCHEMA-003.d.ts +10 -0
- package/dist/rules/schema/SCHEMA-003.js +66 -0
- package/dist/rules/schema/SCHEMA-003.js.map +1 -0
- package/dist/rules/schema/SCHEMA-011.d.ts +10 -0
- package/dist/rules/schema/SCHEMA-011.js +62 -0
- package/dist/rules/schema/SCHEMA-011.js.map +1 -0
- package/dist/rules/security/SEC-001.d.ts +12 -0
- package/dist/rules/security/SEC-001.js +66 -0
- package/dist/rules/security/SEC-001.js.map +1 -0
- package/dist/rules/utils/keywords.d.ts +35 -0
- package/dist/rules/utils/keywords.js +100 -0
- package/dist/rules/utils/keywords.js.map +1 -0
- package/dist/scoring/calculator.d.ts +27 -0
- package/dist/scoring/calculator.js +194 -0
- package/dist/scoring/calculator.js.map +1 -0
- package/dist/scoring/grades.d.ts +34 -0
- package/dist/scoring/grades.js +167 -0
- package/dist/scoring/grades.js.map +1 -0
- package/dist/scoring/index.d.ts +5 -0
- package/dist/scoring/index.js +6 -0
- package/dist/scoring/index.js.map +1 -0
- package/dist/ui/banner.d.ts +21 -0
- package/dist/ui/banner.js +60 -0
- package/dist/ui/banner.js.map +1 -0
- package/dist/ui/design-tokens.d.ts +23 -0
- package/dist/ui/design-tokens.js +58 -0
- package/dist/ui/design-tokens.js.map +1 -0
- package/dist/ui/findings.d.ts +23 -0
- package/dist/ui/findings.js +190 -0
- package/dist/ui/findings.js.map +1 -0
- package/dist/ui/index.d.ts +9 -0
- package/dist/ui/index.js +10 -0
- package/dist/ui/index.js.map +1 -0
- package/dist/ui/ink/App.d.ts +14 -0
- package/dist/ui/ink/App.js +113 -0
- package/dist/ui/ink/App.js.map +1 -0
- package/dist/ui/ink/FullScreenLayout.d.ts +16 -0
- package/dist/ui/ink/FullScreenLayout.js +29 -0
- package/dist/ui/ink/FullScreenLayout.js.map +1 -0
- package/dist/ui/ink/InteractiveApp.d.ts +28 -0
- package/dist/ui/ink/InteractiveApp.js +229 -0
- package/dist/ui/ink/InteractiveApp.js.map +1 -0
- package/dist/ui/ink/RealAuditApp.d.ts +19 -0
- package/dist/ui/ink/RealAuditApp.js +170 -0
- package/dist/ui/ink/RealAuditApp.js.map +1 -0
- package/dist/ui/ink/components/AnimatedProgressBar.d.ts +20 -0
- package/dist/ui/ink/components/AnimatedProgressBar.js +46 -0
- package/dist/ui/ink/components/AnimatedProgressBar.js.map +1 -0
- package/dist/ui/ink/components/AsciiLogo.d.ts +12 -0
- package/dist/ui/ink/components/AsciiLogo.js +35 -0
- package/dist/ui/ink/components/AsciiLogo.js.map +1 -0
- package/dist/ui/ink/components/CategoryBars.d.ts +18 -0
- package/dist/ui/ink/components/CategoryBars.js +18 -0
- package/dist/ui/ink/components/CategoryBars.js.map +1 -0
- package/dist/ui/ink/components/FindingsTable.d.ts +18 -0
- package/dist/ui/ink/components/FindingsTable.js +19 -0
- package/dist/ui/ink/components/FindingsTable.js.map +1 -0
- package/dist/ui/ink/components/Footer.d.ts +15 -0
- package/dist/ui/ink/components/Footer.js +20 -0
- package/dist/ui/ink/components/Footer.js.map +1 -0
- package/dist/ui/ink/components/Header.d.ts +11 -0
- package/dist/ui/ink/components/Header.js +12 -0
- package/dist/ui/ink/components/Header.js.map +1 -0
- package/dist/ui/ink/components/LinkList.d.ts +17 -0
- package/dist/ui/ink/components/LinkList.js +44 -0
- package/dist/ui/ink/components/LinkList.js.map +1 -0
- package/dist/ui/ink/components/Navigation.d.ts +26 -0
- package/dist/ui/ink/components/Navigation.js +62 -0
- package/dist/ui/ink/components/Navigation.js.map +1 -0
- package/dist/ui/ink/components/ProgressBar.d.ts +15 -0
- package/dist/ui/ink/components/ProgressBar.js +14 -0
- package/dist/ui/ink/components/ProgressBar.js.map +1 -0
- package/dist/ui/ink/components/ScoreCard.d.ts +30 -0
- package/dist/ui/ink/components/ScoreCard.js +26 -0
- package/dist/ui/ink/components/ScoreCard.js.map +1 -0
- package/dist/ui/ink/components/SimulationResults.d.ts +33 -0
- package/dist/ui/ink/components/SimulationResults.js +23 -0
- package/dist/ui/ink/components/SimulationResults.js.map +1 -0
- package/dist/ui/ink/components/Spinner.d.ts +11 -0
- package/dist/ui/ink/components/Spinner.js +12 -0
- package/dist/ui/ink/components/Spinner.js.map +1 -0
- package/dist/ui/ink/components/ToolCard.d.ts +23 -0
- package/dist/ui/ink/components/ToolCard.js +20 -0
- package/dist/ui/ink/components/ToolCard.js.map +1 -0
- package/dist/ui/ink/components/shared/Badge.d.ts +21 -0
- package/dist/ui/ink/components/shared/Badge.js +39 -0
- package/dist/ui/ink/components/shared/Badge.js.map +1 -0
- package/dist/ui/ink/components/shared/Card.d.ts +18 -0
- package/dist/ui/ink/components/shared/Card.js +11 -0
- package/dist/ui/ink/components/shared/Card.js.map +1 -0
- package/dist/ui/ink/components/shared/HelpOverlay.d.ts +10 -0
- package/dist/ui/ink/components/shared/HelpOverlay.js +28 -0
- package/dist/ui/ink/components/shared/HelpOverlay.js.map +1 -0
- package/dist/ui/ink/components/shared/LoadingWithTimeout.d.ts +11 -0
- package/dist/ui/ink/components/shared/LoadingWithTimeout.js +21 -0
- package/dist/ui/ink/components/shared/LoadingWithTimeout.js.map +1 -0
- package/dist/ui/ink/components/shared/Menu.d.ts +23 -0
- package/dist/ui/ink/components/shared/Menu.js +43 -0
- package/dist/ui/ink/components/shared/Menu.js.map +1 -0
- package/dist/ui/ink/components/shared/Table.d.ts +23 -0
- package/dist/ui/ink/components/shared/Table.js +40 -0
- package/dist/ui/ink/components/shared/Table.js.map +1 -0
- package/dist/ui/ink/components/views/CrawlingView.d.ts +12 -0
- package/dist/ui/ink/components/views/CrawlingView.js +34 -0
- package/dist/ui/ink/components/views/CrawlingView.js.map +1 -0
- package/dist/ui/ink/components/views/DashboardView.d.ts +21 -0
- package/dist/ui/ink/components/views/DashboardView.js +51 -0
- package/dist/ui/ink/components/views/DashboardView.js.map +1 -0
- package/dist/ui/ink/components/views/FindingDetailView.d.ts +16 -0
- package/dist/ui/ink/components/views/FindingDetailView.js +34 -0
- package/dist/ui/ink/components/views/FindingDetailView.js.map +1 -0
- package/dist/ui/ink/components/views/FindingsView.d.ts +16 -0
- package/dist/ui/ink/components/views/FindingsView.js +79 -0
- package/dist/ui/ink/components/views/FindingsView.js.map +1 -0
- package/dist/ui/ink/components/views/OnboardingView.d.ts +12 -0
- package/dist/ui/ink/components/views/OnboardingView.js +40 -0
- package/dist/ui/ink/components/views/OnboardingView.js.map +1 -0
- package/dist/ui/ink/components/views/SimulationView.d.ts +17 -0
- package/dist/ui/ink/components/views/SimulationView.js +53 -0
- package/dist/ui/ink/components/views/SimulationView.js.map +1 -0
- package/dist/ui/ink/components/views/TestCaseDetailView.d.ts +11 -0
- package/dist/ui/ink/components/views/TestCaseDetailView.js +53 -0
- package/dist/ui/ink/components/views/TestCaseDetailView.js.map +1 -0
- package/dist/ui/ink/components/views/ToolDetailView.d.ts +15 -0
- package/dist/ui/ink/components/views/ToolDetailView.js +25 -0
- package/dist/ui/ink/components/views/ToolDetailView.js.map +1 -0
- package/dist/ui/ink/components/views/ToolsView.d.ts +15 -0
- package/dist/ui/ink/components/views/ToolsView.js +43 -0
- package/dist/ui/ink/components/views/ToolsView.js.map +1 -0
- package/dist/ui/ink/demo.d.ts +6 -0
- package/dist/ui/ink/demo.js +254 -0
- package/dist/ui/ink/demo.js.map +1 -0
- package/dist/ui/ink/hooks/useAnimation.d.ts +29 -0
- package/dist/ui/ink/hooks/useAnimation.js +89 -0
- package/dist/ui/ink/hooks/useAnimation.js.map +1 -0
- package/dist/ui/ink/hooks/useAudit.d.ts +69 -0
- package/dist/ui/ink/hooks/useAudit.js +99 -0
- package/dist/ui/ink/hooks/useAudit.js.map +1 -0
- package/dist/ui/ink/hooks/useCrawlAnimation.d.ts +19 -0
- package/dist/ui/ink/hooks/useCrawlAnimation.js +204 -0
- package/dist/ui/ink/hooks/useCrawlAnimation.js.map +1 -0
- package/dist/ui/ink/hooks/useKeyboardNav.d.ts +23 -0
- package/dist/ui/ink/hooks/useKeyboardNav.js +81 -0
- package/dist/ui/ink/hooks/useKeyboardNav.js.map +1 -0
- package/dist/ui/ink/hooks/useNavigation.d.ts +16 -0
- package/dist/ui/ink/hooks/useNavigation.js +42 -0
- package/dist/ui/ink/hooks/useNavigation.js.map +1 -0
- package/dist/ui/ink/hooks/useTerminalSize.d.ts +10 -0
- package/dist/ui/ink/hooks/useTerminalSize.js +29 -0
- package/dist/ui/ink/hooks/useTerminalSize.js.map +1 -0
- package/dist/ui/ink/index.d.ts +43 -0
- package/dist/ui/ink/index.js +50 -0
- package/dist/ui/ink/index.js.map +1 -0
- package/dist/ui/ink/render.d.ts +24 -0
- package/dist/ui/ink/render.js +14 -0
- package/dist/ui/ink/render.js.map +1 -0
- package/dist/ui/ink/theme.d.ts +37 -0
- package/dist/ui/ink/theme.js +38 -0
- package/dist/ui/ink/theme.js.map +1 -0
- package/dist/ui/ink/types.d.ts +77 -0
- package/dist/ui/ink/types.js +5 -0
- package/dist/ui/ink/types.js.map +1 -0
- package/dist/ui/score-display.d.ts +16 -0
- package/dist/ui/score-display.js +201 -0
- package/dist/ui/score-display.js.map +1 -0
- package/dist/ui/spinner.d.ts +45 -0
- package/dist/ui/spinner.js +112 -0
- package/dist/ui/spinner.js.map +1 -0
- package/dist/ui/utils.d.ts +13 -0
- package/dist/ui/utils.js +25 -0
- package/dist/ui/utils.js.map +1 -0
- package/package.json +61 -9
- package/index.js +0 -105
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rules Module
|
|
3
|
+
*
|
|
4
|
+
* Registers all audit rules.
|
|
5
|
+
*/
|
|
6
|
+
import { IMP_001 } from './implementation/IMP-001.js';
|
|
7
|
+
import { IMP_003 } from './implementation/IMP-003.js';
|
|
8
|
+
import { IMP_004 } from './implementation/IMP-004.js';
|
|
9
|
+
import { IMP_005 } from './implementation/IMP-005.js';
|
|
10
|
+
import { IMP_007 } from './implementation/IMP-007.js';
|
|
11
|
+
import { IMP_013 } from './implementation/IMP-013.js';
|
|
12
|
+
import { DESC_003 } from './description/DESC-003.js';
|
|
13
|
+
import { DESC_004 } from './description/DESC-004.js';
|
|
14
|
+
import { DESC_005 } from './description/DESC-005.js';
|
|
15
|
+
import { SCHEMA_001 } from './schema/SCHEMA-001.js';
|
|
16
|
+
import { SCHEMA_002 } from './schema/SCHEMA-002.js';
|
|
17
|
+
import { SCHEMA_003 } from './schema/SCHEMA-003.js';
|
|
18
|
+
import { SCHEMA_011 } from './schema/SCHEMA-011.js';
|
|
19
|
+
import { SEC_001 } from './security/SEC-001.js';
|
|
20
|
+
import { BP_001 } from './best-practices/BP-001.js';
|
|
21
|
+
import { BP_002 } from './best-practices/BP-002.js';
|
|
22
|
+
import { BP_003 } from './best-practices/BP-003.js';
|
|
23
|
+
import { COV_001 } from './coverage/COV-001.js';
|
|
24
|
+
import { LLM_001 } from './llm/LLM-001.js';
|
|
25
|
+
import { LLM_002 } from './llm/LLM-002.js';
|
|
26
|
+
import { LLM_003 } from './llm/LLM-003.js';
|
|
27
|
+
import { LLM_004 } from './llm/LLM-004.js';
|
|
28
|
+
import { LLM_005 } from './llm/LLM-005.js';
|
|
29
|
+
/**
|
|
30
|
+
* Register all rules
|
|
31
|
+
*/
|
|
32
|
+
export declare function registerAllRules(): void;
|
|
33
|
+
/**
|
|
34
|
+
* Register all rules once per process.
|
|
35
|
+
*
|
|
36
|
+
* The rule runner rejects duplicate IDs, so repeated runtime audits should
|
|
37
|
+
* use this helper instead of calling registerAllRules() directly.
|
|
38
|
+
*/
|
|
39
|
+
export declare function ensureRulesRegistered(): void;
|
|
40
|
+
/**
|
|
41
|
+
* Clear all registered rules (for testing/dev resets).
|
|
42
|
+
*/
|
|
43
|
+
export declare function clearRules(): void;
|
|
44
|
+
export { registerRule, getRule, getAllRules, getRulesByCategory, getRuleCategories, getRuleSeverities, runAllRules, runRules, createRuleResult, } from './runner.js';
|
|
45
|
+
export { IMP_001, IMP_003, IMP_004, IMP_005, IMP_007, IMP_013 };
|
|
46
|
+
export { DESC_003, DESC_004, DESC_005 };
|
|
47
|
+
export { SCHEMA_001, SCHEMA_002, SCHEMA_003, SCHEMA_011 };
|
|
48
|
+
export { SEC_001 };
|
|
49
|
+
export { BP_001, BP_002, BP_003 };
|
|
50
|
+
export { COV_001 };
|
|
51
|
+
export { LLM_001, LLM_002, LLM_003, LLM_004, LLM_005 };
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rules Module
|
|
3
|
+
*
|
|
4
|
+
* Registers all audit rules.
|
|
5
|
+
*/
|
|
6
|
+
import { registerRule, getAllRules, clearRules as clearRegisteredRules, } from './runner.js';
|
|
7
|
+
// Implementation rules
|
|
8
|
+
import { IMP_001 } from './implementation/IMP-001.js';
|
|
9
|
+
import { IMP_003 } from './implementation/IMP-003.js';
|
|
10
|
+
import { IMP_004 } from './implementation/IMP-004.js';
|
|
11
|
+
import { IMP_005 } from './implementation/IMP-005.js';
|
|
12
|
+
import { IMP_007 } from './implementation/IMP-007.js';
|
|
13
|
+
import { IMP_013 } from './implementation/IMP-013.js';
|
|
14
|
+
// Description rules
|
|
15
|
+
import { DESC_003 } from './description/DESC-003.js';
|
|
16
|
+
import { DESC_004 } from './description/DESC-004.js';
|
|
17
|
+
import { DESC_005 } from './description/DESC-005.js';
|
|
18
|
+
// Schema rules
|
|
19
|
+
import { SCHEMA_001 } from './schema/SCHEMA-001.js';
|
|
20
|
+
import { SCHEMA_002 } from './schema/SCHEMA-002.js';
|
|
21
|
+
import { SCHEMA_003 } from './schema/SCHEMA-003.js';
|
|
22
|
+
import { SCHEMA_011 } from './schema/SCHEMA-011.js';
|
|
23
|
+
// Security rules
|
|
24
|
+
import { SEC_001 } from './security/SEC-001.js';
|
|
25
|
+
// Best practices rules
|
|
26
|
+
import { BP_001 } from './best-practices/BP-001.js';
|
|
27
|
+
import { BP_002 } from './best-practices/BP-002.js';
|
|
28
|
+
import { BP_003 } from './best-practices/BP-003.js';
|
|
29
|
+
// Coverage rules
|
|
30
|
+
import { COV_001 } from './coverage/COV-001.js';
|
|
31
|
+
// LLM-powered rules
|
|
32
|
+
import { LLM_001 } from './llm/LLM-001.js';
|
|
33
|
+
import { LLM_002 } from './llm/LLM-002.js';
|
|
34
|
+
import { LLM_003 } from './llm/LLM-003.js';
|
|
35
|
+
import { LLM_004 } from './llm/LLM-004.js';
|
|
36
|
+
import { LLM_005 } from './llm/LLM-005.js';
|
|
37
|
+
/**
|
|
38
|
+
* Register all rules
|
|
39
|
+
*/
|
|
40
|
+
export function registerAllRules() {
|
|
41
|
+
// Implementation rules
|
|
42
|
+
registerRule(IMP_001);
|
|
43
|
+
registerRule(IMP_003);
|
|
44
|
+
registerRule(IMP_004);
|
|
45
|
+
registerRule(IMP_005);
|
|
46
|
+
registerRule(IMP_007);
|
|
47
|
+
registerRule(IMP_013);
|
|
48
|
+
// Description rules
|
|
49
|
+
registerRule(DESC_003);
|
|
50
|
+
registerRule(DESC_004);
|
|
51
|
+
registerRule(DESC_005);
|
|
52
|
+
// Schema rules
|
|
53
|
+
registerRule(SCHEMA_001);
|
|
54
|
+
registerRule(SCHEMA_002);
|
|
55
|
+
registerRule(SCHEMA_003);
|
|
56
|
+
registerRule(SCHEMA_011);
|
|
57
|
+
// Security rules
|
|
58
|
+
registerRule(SEC_001);
|
|
59
|
+
// Best practices rules
|
|
60
|
+
registerRule(BP_001);
|
|
61
|
+
registerRule(BP_002);
|
|
62
|
+
registerRule(BP_003);
|
|
63
|
+
// Coverage rules
|
|
64
|
+
registerRule(COV_001);
|
|
65
|
+
// LLM-powered rules (gracefully skip if no API key)
|
|
66
|
+
registerRule(LLM_001);
|
|
67
|
+
registerRule(LLM_002);
|
|
68
|
+
registerRule(LLM_003);
|
|
69
|
+
registerRule(LLM_004);
|
|
70
|
+
registerRule(LLM_005);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Register all rules once per process.
|
|
74
|
+
*
|
|
75
|
+
* The rule runner rejects duplicate IDs, so repeated runtime audits should
|
|
76
|
+
* use this helper instead of calling registerAllRules() directly.
|
|
77
|
+
*/
|
|
78
|
+
export function ensureRulesRegistered() {
|
|
79
|
+
if (getAllRules().length > 0) {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
registerAllRules();
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Clear all registered rules (for testing/dev resets).
|
|
86
|
+
*/
|
|
87
|
+
export function clearRules() {
|
|
88
|
+
clearRegisteredRules();
|
|
89
|
+
}
|
|
90
|
+
// Export runner functions
|
|
91
|
+
export { registerRule, getRule, getAllRules, getRulesByCategory, getRuleCategories, getRuleSeverities, runAllRules, runRules, createRuleResult, } from './runner.js';
|
|
92
|
+
// Export individual rules for direct access
|
|
93
|
+
export { IMP_001, IMP_003, IMP_004, IMP_005, IMP_007, IMP_013 };
|
|
94
|
+
export { DESC_003, DESC_004, DESC_005 };
|
|
95
|
+
export { SCHEMA_001, SCHEMA_002, SCHEMA_003, SCHEMA_011 };
|
|
96
|
+
export { SEC_001 };
|
|
97
|
+
export { BP_001, BP_002, BP_003 };
|
|
98
|
+
export { COV_001 };
|
|
99
|
+
export { LLM_001, LLM_002, LLM_003, LLM_004, LLM_005 };
|
|
100
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/rules/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,YAAY,EACZ,WAAW,EACX,UAAU,IAAI,oBAAoB,GACnC,MAAM,aAAa,CAAC;AAErB,uBAAuB;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAEtD,oBAAoB;AACpB,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAErD,eAAe;AACf,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,iBAAiB;AACjB,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,uBAAuB;AACvB,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAEpD,iBAAiB;AACjB,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,oBAAoB;AACpB,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAC9B,uBAAuB;IACvB,YAAY,CAAC,OAAO,CAAC,CAAC;IACtB,YAAY,CAAC,OAAO,CAAC,CAAC;IACtB,YAAY,CAAC,OAAO,CAAC,CAAC;IACtB,YAAY,CAAC,OAAO,CAAC,CAAC;IACtB,YAAY,CAAC,OAAO,CAAC,CAAC;IACtB,YAAY,CAAC,OAAO,CAAC,CAAC;IAEtB,oBAAoB;IACpB,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvB,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvB,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEvB,eAAe;IACf,YAAY,CAAC,UAAU,CAAC,CAAC;IACzB,YAAY,CAAC,UAAU,CAAC,CAAC;IACzB,YAAY,CAAC,UAAU,CAAC,CAAC;IACzB,YAAY,CAAC,UAAU,CAAC,CAAC;IAEzB,iBAAiB;IACjB,YAAY,CAAC,OAAO,CAAC,CAAC;IAEtB,uBAAuB;IACvB,YAAY,CAAC,MAAM,CAAC,CAAC;IACrB,YAAY,CAAC,MAAM,CAAC,CAAC;IACrB,YAAY,CAAC,MAAM,CAAC,CAAC;IAErB,iBAAiB;IACjB,YAAY,CAAC,OAAO,CAAC,CAAC;IAEtB,oDAAoD;IACpD,YAAY,CAAC,OAAO,CAAC,CAAC;IACtB,YAAY,CAAC,OAAO,CAAC,CAAC;IACtB,YAAY,CAAC,OAAO,CAAC,CAAC;IACtB,YAAY,CAAC,OAAO,CAAC,CAAC;IACtB,YAAY,CAAC,OAAO,CAAC,CAAC;AACxB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB;IACnC,IAAI,WAAW,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO;IACT,CAAC;IACD,gBAAgB,EAAE,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU;IACxB,oBAAoB,EAAE,CAAC;AACzB,CAAC;AAED,0BAA0B;AAC1B,OAAO,EACL,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,WAAW,EACX,QAAQ,EACR,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAErB,4CAA4C;AAC5C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,CAAC;AACnB,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,CAAC;AACnB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-001: Description Steering Quality
|
|
3
|
+
*
|
|
4
|
+
* LLM-based evaluation of how well the tool description will
|
|
5
|
+
* guide AI agents to use the tool correctly.
|
|
6
|
+
*
|
|
7
|
+
* Evaluates:
|
|
8
|
+
* - Will AI pick this tool at the right time? (routing accuracy)
|
|
9
|
+
* - Will AI fill parameters correctly? (parameter accuracy)
|
|
10
|
+
* - Is description specific enough to avoid hallucination?
|
|
11
|
+
*/
|
|
12
|
+
import type { Rule } from '../../core/types/rule.js';
|
|
13
|
+
export declare const LLM_001: Rule;
|
|
14
|
+
export default LLM_001;
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-001: Description Steering Quality
|
|
3
|
+
*
|
|
4
|
+
* LLM-based evaluation of how well the tool description will
|
|
5
|
+
* guide AI agents to use the tool correctly.
|
|
6
|
+
*
|
|
7
|
+
* Evaluates:
|
|
8
|
+
* - Will AI pick this tool at the right time? (routing accuracy)
|
|
9
|
+
* - Will AI fill parameters correctly? (parameter accuracy)
|
|
10
|
+
* - Is description specific enough to avoid hallucination?
|
|
11
|
+
*/
|
|
12
|
+
import { createRuleResult } from '../runner.js';
|
|
13
|
+
import { getDefaultEvaluator } from '../../llm/evaluator.js';
|
|
14
|
+
export const LLM_001 = {
|
|
15
|
+
id: 'LLM-001',
|
|
16
|
+
category: 'description',
|
|
17
|
+
name: 'Description Steering Quality',
|
|
18
|
+
description: 'LLM evaluation of how well the description guides AI agents',
|
|
19
|
+
severity: 'warning',
|
|
20
|
+
maxScore: 15,
|
|
21
|
+
async check(context) {
|
|
22
|
+
const tools = context.tools;
|
|
23
|
+
if (tools.length === 0) {
|
|
24
|
+
return createRuleResult('LLM-001', 15, {
|
|
25
|
+
passed: true,
|
|
26
|
+
score: 15,
|
|
27
|
+
message: 'No tools detected (rule not applicable)',
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
const evaluator = getDefaultEvaluator();
|
|
31
|
+
// Check if LLM is available
|
|
32
|
+
if (!evaluator.isLLMAvailable()) {
|
|
33
|
+
return createRuleResult('LLM-001', 15, {
|
|
34
|
+
passed: true,
|
|
35
|
+
score: 15,
|
|
36
|
+
message: 'LLM evaluation skipped (no API key configured)',
|
|
37
|
+
details: ['Set OPENROUTER_API_KEY for LLM-based analysis'],
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
const issues = [];
|
|
41
|
+
let totalScore = 0;
|
|
42
|
+
for (const tool of tools) {
|
|
43
|
+
const result = await evaluator.runEvaluation(tool, 'steering-quality');
|
|
44
|
+
if (!result.passed || result.score < 7) {
|
|
45
|
+
issues.push({
|
|
46
|
+
tool: tool.name,
|
|
47
|
+
score: result.score,
|
|
48
|
+
reasoning: result.reasoning,
|
|
49
|
+
suggestions: result.suggestions || [],
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
totalScore += result.score;
|
|
53
|
+
}
|
|
54
|
+
const avgScore = totalScore / tools.length;
|
|
55
|
+
const normalizedScore = Math.round((avgScore / 10) * 15);
|
|
56
|
+
if (issues.length === 0) {
|
|
57
|
+
return createRuleResult('LLM-001', 15, {
|
|
58
|
+
passed: true,
|
|
59
|
+
score: normalizedScore,
|
|
60
|
+
message: 'All tool descriptions provide good steering for AI agents',
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
return createRuleResult('LLM-001', 15, {
|
|
64
|
+
passed: false,
|
|
65
|
+
score: normalizedScore,
|
|
66
|
+
message: `${issues.length} tool(s) have descriptions that may confuse AI agents`,
|
|
67
|
+
details: issues.map((i) => `Tool "${i.tool}" (score: ${i.score}/10): ${i.reasoning}`),
|
|
68
|
+
suggestions: [
|
|
69
|
+
...issues.flatMap((i) => i.suggestions),
|
|
70
|
+
'Good descriptions clearly state WHAT the tool does and WHEN to use it',
|
|
71
|
+
'Include key parameter expectations in the description',
|
|
72
|
+
].filter((s, i, arr) => arr.indexOf(s) === i).slice(0, 5),
|
|
73
|
+
affectedTools: issues.map((i) => i.tool),
|
|
74
|
+
});
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
export default LLM_001;
|
|
78
|
+
//# sourceMappingURL=LLM-001.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LLM-001.js","sourceRoot":"","sources":["../../../src/rules/llm/LLM-001.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,MAAM,CAAC,MAAM,OAAO,GAAS;IAC3B,EAAE,EAAE,SAAS;IACb,QAAQ,EAAE,aAAa;IACvB,IAAI,EAAE,8BAA8B;IACpC,WAAW,EAAE,6DAA6D;IAC1E,QAAQ,EAAE,SAAS;IACnB,QAAQ,EAAE,EAAE;IAEZ,KAAK,CAAC,KAAK,CAAC,OAAoB;QAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAE5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,gBAAgB,CAAC,SAAS,EAAE,EAAE,EAAE;gBACrC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,EAAE;gBACT,OAAO,EAAE,yCAAyC;aACnD,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,mBAAmB,EAAE,CAAC;QAExC,4BAA4B;QAC5B,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,CAAC;YAChC,OAAO,gBAAgB,CAAC,SAAS,EAAE,EAAE,EAAE;gBACrC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,EAAE;gBACT,OAAO,EAAE,gDAAgD;gBACzD,OAAO,EAAE,CAAC,+CAA+C,CAAC;aAC3D,CAAC,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAgF,EAAE,CAAC;QAC/F,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YAEvE,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBACvC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;iBACtC,CAAC,CAAC;YACL,CAAC;YAED,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC;QAC7B,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAEzD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,gBAAgB,CAAC,SAAS,EAAE,EAAE,EAAE;gBACrC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE,2DAA2D;aACrE,CAAC,CAAC;QACL,CAAC;QAED,OAAO,gBAAgB,CAAC,SAAS,EAAE,EAAE,EAAE;YACrC,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,eAAe;YACtB,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,uDAAuD;YAChF,OAAO,EAAE,MAAM,CAAC,GAAG,CACjB,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,SAAS,EAAE,CACjE;YACD,WAAW,EAAE;gBACX,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;gBACvC,uEAAuE;gBACvE,uDAAuD;aACxD,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YACzD,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACzC,CAAC,CAAC;IACL,CAAC;CACF,CAAC;AAEF,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-002: Tool Atomicity Check
|
|
3
|
+
*
|
|
4
|
+
* LLM-based evaluation of whether tools are atomic (single function)
|
|
5
|
+
* or compound (doing multiple unrelated things).
|
|
6
|
+
*
|
|
7
|
+
* Atomic tools are easier for AI agents to:
|
|
8
|
+
* - Understand when to use
|
|
9
|
+
* - Route correctly
|
|
10
|
+
* - Compose with other tools
|
|
11
|
+
*/
|
|
12
|
+
import type { Rule } from '../../core/types/rule.js';
|
|
13
|
+
export declare const LLM_002: Rule;
|
|
14
|
+
export default LLM_002;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-002: Tool Atomicity Check
|
|
3
|
+
*
|
|
4
|
+
* LLM-based evaluation of whether tools are atomic (single function)
|
|
5
|
+
* or compound (doing multiple unrelated things).
|
|
6
|
+
*
|
|
7
|
+
* Atomic tools are easier for AI agents to:
|
|
8
|
+
* - Understand when to use
|
|
9
|
+
* - Route correctly
|
|
10
|
+
* - Compose with other tools
|
|
11
|
+
*/
|
|
12
|
+
import { createRuleResult } from '../runner.js';
|
|
13
|
+
import { getDefaultEvaluator } from '../../llm/evaluator.js';
|
|
14
|
+
export const LLM_002 = {
|
|
15
|
+
id: 'LLM-002',
|
|
16
|
+
category: 'best-practices',
|
|
17
|
+
name: 'Tool Atomicity',
|
|
18
|
+
description: 'LLM evaluation of whether tools do one thing well',
|
|
19
|
+
severity: 'warning',
|
|
20
|
+
maxScore: 10,
|
|
21
|
+
async check(context) {
|
|
22
|
+
const tools = context.tools;
|
|
23
|
+
if (tools.length === 0) {
|
|
24
|
+
return createRuleResult('LLM-002', 10, {
|
|
25
|
+
passed: true,
|
|
26
|
+
score: 10,
|
|
27
|
+
message: 'No tools detected (rule not applicable)',
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
const evaluator = getDefaultEvaluator();
|
|
31
|
+
if (!evaluator.isLLMAvailable()) {
|
|
32
|
+
return createRuleResult('LLM-002', 10, {
|
|
33
|
+
passed: true,
|
|
34
|
+
score: 10,
|
|
35
|
+
message: 'LLM evaluation skipped (no API key configured)',
|
|
36
|
+
details: ['Set OPENROUTER_API_KEY for LLM-based analysis'],
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
const nonAtomicTools = [];
|
|
40
|
+
let totalScore = 0;
|
|
41
|
+
for (const tool of tools) {
|
|
42
|
+
const result = await evaluator.runEvaluation(tool, 'bp-atomic-tool');
|
|
43
|
+
if (!result.passed || result.score < 7) {
|
|
44
|
+
nonAtomicTools.push({
|
|
45
|
+
tool: tool.name,
|
|
46
|
+
reasoning: result.reasoning,
|
|
47
|
+
suggestions: result.suggestions || [],
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
totalScore += result.score;
|
|
51
|
+
}
|
|
52
|
+
const avgScore = totalScore / tools.length;
|
|
53
|
+
const normalizedScore = Math.round((avgScore / 10) * 10);
|
|
54
|
+
if (nonAtomicTools.length === 0) {
|
|
55
|
+
return createRuleResult('LLM-002', 10, {
|
|
56
|
+
passed: true,
|
|
57
|
+
score: normalizedScore,
|
|
58
|
+
message: 'All tools appear to be atomic (single function)',
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
return createRuleResult('LLM-002', 10, {
|
|
62
|
+
passed: false,
|
|
63
|
+
score: normalizedScore,
|
|
64
|
+
message: `${nonAtomicTools.length} tool(s) may be doing too many things`,
|
|
65
|
+
details: nonAtomicTools.map((t) => `Tool "${t.tool}": ${t.reasoning}`),
|
|
66
|
+
suggestions: [
|
|
67
|
+
'Split compound tools into smaller, focused tools',
|
|
68
|
+
'Each tool should have a single, clear purpose',
|
|
69
|
+
'Atomic tools are easier for AI agents to use correctly',
|
|
70
|
+
...nonAtomicTools.flatMap((t) => t.suggestions),
|
|
71
|
+
].filter((s, i, arr) => arr.indexOf(s) === i).slice(0, 5),
|
|
72
|
+
affectedTools: nonAtomicTools.map((t) => t.tool),
|
|
73
|
+
});
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
export default LLM_002;
|
|
77
|
+
//# sourceMappingURL=LLM-002.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LLM-002.js","sourceRoot":"","sources":["../../../src/rules/llm/LLM-002.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,MAAM,CAAC,MAAM,OAAO,GAAS;IAC3B,EAAE,EAAE,SAAS;IACb,QAAQ,EAAE,gBAAgB;IAC1B,IAAI,EAAE,gBAAgB;IACtB,WAAW,EAAE,mDAAmD;IAChE,QAAQ,EAAE,SAAS;IACnB,QAAQ,EAAE,EAAE;IAEZ,KAAK,CAAC,KAAK,CAAC,OAAoB;QAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAE5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,gBAAgB,CAAC,SAAS,EAAE,EAAE,EAAE;gBACrC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,EAAE;gBACT,OAAO,EAAE,yCAAyC;aACnD,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,mBAAmB,EAAE,CAAC;QAExC,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,CAAC;YAChC,OAAO,gBAAgB,CAAC,SAAS,EAAE,EAAE,EAAE;gBACrC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,EAAE;gBACT,OAAO,EAAE,gDAAgD;gBACzD,OAAO,EAAE,CAAC,+CAA+C,CAAC;aAC3D,CAAC,CAAC;QACL,CAAC;QAED,MAAM,cAAc,GAAiE,EAAE,CAAC;QACxF,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;YAErE,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBACvC,cAAc,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;iBACtC,CAAC,CAAC;YACL,CAAC;YAED,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC;QAC7B,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAEzD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO,gBAAgB,CAAC,SAAS,EAAE,EAAE,EAAE;gBACrC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE,iDAAiD;aAC3D,CAAC,CAAC;QACL,CAAC;QAED,OAAO,gBAAgB,CAAC,SAAS,EAAE,EAAE,EAAE;YACrC,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,eAAe;YACtB,OAAO,EAAE,GAAG,cAAc,CAAC,MAAM,uCAAuC;YACxE,OAAO,EAAE,cAAc,CAAC,GAAG,CACzB,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,SAAS,EAAE,CAC1C;YACD,WAAW,EAAE;gBACX,kDAAkD;gBAClD,+CAA+C;gBAC/C,wDAAwD;gBACxD,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;aAChD,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YACzD,aAAa,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACjD,CAAC,CAAC;IACL,CAAC;CACF,CAAC;AAEF,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-003: Positive Framing Check
|
|
3
|
+
*
|
|
4
|
+
* LLM-based evaluation of whether descriptions use positive
|
|
5
|
+
* instructions rather than negative limitations.
|
|
6
|
+
*
|
|
7
|
+
* Good: "Search for flights between two airports"
|
|
8
|
+
* Bad: "Do not use this for hotel searches"
|
|
9
|
+
*
|
|
10
|
+
* Negative framing can confuse AI agents and lead to:
|
|
11
|
+
* - Misrouting (avoiding tools they should use)
|
|
12
|
+
* - Confusion about tool capabilities
|
|
13
|
+
*/
|
|
14
|
+
import type { Rule } from '../../core/types/rule.js';
|
|
15
|
+
export declare const LLM_003: Rule;
|
|
16
|
+
export default LLM_003;
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-003: Positive Framing Check
|
|
3
|
+
*
|
|
4
|
+
* LLM-based evaluation of whether descriptions use positive
|
|
5
|
+
* instructions rather than negative limitations.
|
|
6
|
+
*
|
|
7
|
+
* Good: "Search for flights between two airports"
|
|
8
|
+
* Bad: "Do not use this for hotel searches"
|
|
9
|
+
*
|
|
10
|
+
* Negative framing can confuse AI agents and lead to:
|
|
11
|
+
* - Misrouting (avoiding tools they should use)
|
|
12
|
+
* - Confusion about tool capabilities
|
|
13
|
+
*/
|
|
14
|
+
import { createRuleResult } from '../runner.js';
|
|
15
|
+
import { getDefaultEvaluator } from '../../llm/evaluator.js';
|
|
16
|
+
export const LLM_003 = {
|
|
17
|
+
id: 'LLM-003',
|
|
18
|
+
category: 'description',
|
|
19
|
+
name: 'Positive Framing',
|
|
20
|
+
description: 'LLM evaluation of positive vs negative instruction framing',
|
|
21
|
+
severity: 'info',
|
|
22
|
+
maxScore: 8,
|
|
23
|
+
async check(context) {
|
|
24
|
+
const tools = context.tools;
|
|
25
|
+
if (tools.length === 0) {
|
|
26
|
+
return createRuleResult('LLM-003', 8, {
|
|
27
|
+
passed: true,
|
|
28
|
+
score: 8,
|
|
29
|
+
message: 'No tools detected (rule not applicable)',
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
const evaluator = getDefaultEvaluator();
|
|
33
|
+
if (!evaluator.isLLMAvailable()) {
|
|
34
|
+
return createRuleResult('LLM-003', 8, {
|
|
35
|
+
passed: true,
|
|
36
|
+
score: 8,
|
|
37
|
+
message: 'LLM evaluation skipped (no API key configured)',
|
|
38
|
+
details: ['Set OPENROUTER_API_KEY for LLM-based analysis'],
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
const negativeFraming = [];
|
|
42
|
+
let totalScore = 0;
|
|
43
|
+
for (const tool of tools) {
|
|
44
|
+
const result = await evaluator.runEvaluation(tool, 'desc-positive-framing');
|
|
45
|
+
if (!result.passed || result.score < 7) {
|
|
46
|
+
negativeFraming.push({
|
|
47
|
+
tool: tool.name,
|
|
48
|
+
issues: result.issues || [],
|
|
49
|
+
suggestions: result.suggestions || [],
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
totalScore += result.score;
|
|
53
|
+
}
|
|
54
|
+
const avgScore = totalScore / tools.length;
|
|
55
|
+
const normalizedScore = Math.round((avgScore / 10) * 8);
|
|
56
|
+
if (negativeFraming.length === 0) {
|
|
57
|
+
return createRuleResult('LLM-003', 8, {
|
|
58
|
+
passed: true,
|
|
59
|
+
score: normalizedScore,
|
|
60
|
+
message: 'All tool descriptions use positive framing',
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
return createRuleResult('LLM-003', 8, {
|
|
64
|
+
passed: false,
|
|
65
|
+
score: normalizedScore,
|
|
66
|
+
message: `${negativeFraming.length} tool(s) use negative framing in descriptions`,
|
|
67
|
+
details: negativeFraming.flatMap((t) => [
|
|
68
|
+
`Tool "${t.tool}":`,
|
|
69
|
+
...t.issues.map((i) => ` - ${i}`),
|
|
70
|
+
]),
|
|
71
|
+
suggestions: [
|
|
72
|
+
'Rewrite descriptions to explain what the tool DOES, not what it doesn\'t',
|
|
73
|
+
'Avoid: "do not", "don\'t", "never", "avoid", "cannot"',
|
|
74
|
+
'Instead of "Don\'t use for X", say "Use for Y"',
|
|
75
|
+
...negativeFraming.flatMap((t) => t.suggestions),
|
|
76
|
+
].filter((s, i, arr) => arr.indexOf(s) === i).slice(0, 5),
|
|
77
|
+
affectedTools: negativeFraming.map((t) => t.tool),
|
|
78
|
+
});
|
|
79
|
+
},
|
|
80
|
+
};
|
|
81
|
+
export default LLM_003;
|
|
82
|
+
//# sourceMappingURL=LLM-003.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LLM-003.js","sourceRoot":"","sources":["../../../src/rules/llm/LLM-003.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,MAAM,CAAC,MAAM,OAAO,GAAS;IAC3B,EAAE,EAAE,SAAS;IACb,QAAQ,EAAE,aAAa;IACvB,IAAI,EAAE,kBAAkB;IACxB,WAAW,EAAE,4DAA4D;IACzE,QAAQ,EAAE,MAAM;IAChB,QAAQ,EAAE,CAAC;IAEX,KAAK,CAAC,KAAK,CAAC,OAAoB;QAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAE5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,gBAAgB,CAAC,SAAS,EAAE,CAAC,EAAE;gBACpC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,yCAAyC;aACnD,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,mBAAmB,EAAE,CAAC;QAExC,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,CAAC;YAChC,OAAO,gBAAgB,CAAC,SAAS,EAAE,CAAC,EAAE;gBACpC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gDAAgD;gBACzD,OAAO,EAAE,CAAC,+CAA+C,CAAC;aAC3D,CAAC,CAAC;QACL,CAAC;QAED,MAAM,eAAe,GAAgE,EAAE,CAAC;QACxF,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;YAE5E,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBACvC,eAAe,CAAC,IAAI,CAAC;oBACnB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE;oBAC3B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;iBACtC,CAAC,CAAC;YACL,CAAC;YAED,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC;QAC7B,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAExD,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,gBAAgB,CAAC,SAAS,EAAE,CAAC,EAAE;gBACpC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE,4CAA4C;aACtD,CAAC,CAAC;QACL,CAAC;QAED,OAAO,gBAAgB,CAAC,SAAS,EAAE,CAAC,EAAE;YACpC,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,eAAe;YACtB,OAAO,EAAE,GAAG,eAAe,CAAC,MAAM,+CAA+C;YACjF,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;gBACtC,SAAS,CAAC,CAAC,IAAI,IAAI;gBACnB,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;aACnC,CAAC;YACF,WAAW,EAAE;gBACX,0EAA0E;gBAC1E,uDAAuD;gBACvD,gDAAgD;gBAChD,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;aACjD,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YACzD,aAAa,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SAClD,CAAC,CAAC;IACL,CAAC;CACF,CAAC;AAEF,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-004: Description Completeness
|
|
3
|
+
*
|
|
4
|
+
* LLM-based evaluation of whether descriptions adequately
|
|
5
|
+
* explain WHAT the tool does and WHEN to use it.
|
|
6
|
+
*
|
|
7
|
+
* Complete descriptions help AI agents:
|
|
8
|
+
* - Correctly route requests to the right tool
|
|
9
|
+
* - Understand the tool's capabilities and limitations
|
|
10
|
+
* - Know when NOT to use the tool
|
|
11
|
+
*/
|
|
12
|
+
import type { Rule } from '../../core/types/rule.js';
|
|
13
|
+
export declare const LLM_004: Rule;
|
|
14
|
+
export default LLM_004;
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-004: Description Completeness
|
|
3
|
+
*
|
|
4
|
+
* LLM-based evaluation of whether descriptions adequately
|
|
5
|
+
* explain WHAT the tool does and WHEN to use it.
|
|
6
|
+
*
|
|
7
|
+
* Complete descriptions help AI agents:
|
|
8
|
+
* - Correctly route requests to the right tool
|
|
9
|
+
* - Understand the tool's capabilities and limitations
|
|
10
|
+
* - Know when NOT to use the tool
|
|
11
|
+
*/
|
|
12
|
+
import { createRuleResult } from '../runner.js';
|
|
13
|
+
import { getDefaultEvaluator } from '../../llm/evaluator.js';
|
|
14
|
+
export const LLM_004 = {
|
|
15
|
+
id: 'LLM-004',
|
|
16
|
+
category: 'description',
|
|
17
|
+
name: 'Description Completeness',
|
|
18
|
+
description: 'LLM evaluation of whether descriptions explain what and when',
|
|
19
|
+
severity: 'warning',
|
|
20
|
+
maxScore: 10,
|
|
21
|
+
async check(context) {
|
|
22
|
+
const tools = context.tools;
|
|
23
|
+
if (tools.length === 0) {
|
|
24
|
+
return createRuleResult('LLM-004', 10, {
|
|
25
|
+
passed: true,
|
|
26
|
+
score: 10,
|
|
27
|
+
message: 'No tools detected (rule not applicable)',
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
const evaluator = getDefaultEvaluator();
|
|
31
|
+
if (!evaluator.isLLMAvailable()) {
|
|
32
|
+
return createRuleResult('LLM-004', 10, {
|
|
33
|
+
passed: true,
|
|
34
|
+
score: 10,
|
|
35
|
+
message: 'LLM evaluation skipped (no API key configured)',
|
|
36
|
+
details: ['Set OPENROUTER_API_KEY for LLM-based analysis'],
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
const incompleteTools = [];
|
|
40
|
+
let totalScore = 0;
|
|
41
|
+
for (const tool of tools) {
|
|
42
|
+
const result = await evaluator.runEvaluation(tool, 'desc-explains-what');
|
|
43
|
+
if (!result.passed || result.score < 7) {
|
|
44
|
+
// Analyze what's missing
|
|
45
|
+
const reasoning = result.reasoning.toLowerCase();
|
|
46
|
+
incompleteTools.push({
|
|
47
|
+
tool: tool.name,
|
|
48
|
+
missingWhat: reasoning.includes('what') || reasoning.includes('action'),
|
|
49
|
+
missingWhen: reasoning.includes('when') || reasoning.includes('context'),
|
|
50
|
+
suggestions: result.suggestions || [],
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
totalScore += result.score;
|
|
54
|
+
}
|
|
55
|
+
const avgScore = totalScore / tools.length;
|
|
56
|
+
const normalizedScore = Math.round((avgScore / 10) * 10);
|
|
57
|
+
if (incompleteTools.length === 0) {
|
|
58
|
+
return createRuleResult('LLM-004', 10, {
|
|
59
|
+
passed: true,
|
|
60
|
+
score: normalizedScore,
|
|
61
|
+
message: 'All tool descriptions are complete',
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
return createRuleResult('LLM-004', 10, {
|
|
65
|
+
passed: false,
|
|
66
|
+
score: normalizedScore,
|
|
67
|
+
message: `${incompleteTools.length} tool(s) have incomplete descriptions`,
|
|
68
|
+
details: incompleteTools.map((t) => {
|
|
69
|
+
const missing = [];
|
|
70
|
+
if (t.missingWhat)
|
|
71
|
+
missing.push('WHAT it does');
|
|
72
|
+
if (t.missingWhen)
|
|
73
|
+
missing.push('WHEN to use it');
|
|
74
|
+
return `Tool "${t.tool}" is missing: ${missing.join(' and ') || 'clarity'}`;
|
|
75
|
+
}),
|
|
76
|
+
suggestions: [
|
|
77
|
+
'Descriptions should clearly state WHAT the tool does',
|
|
78
|
+
'Include context for WHEN to use the tool',
|
|
79
|
+
'Example: "Search for available flights between airports. Use when user wants to find flight options."',
|
|
80
|
+
...incompleteTools.flatMap((t) => t.suggestions),
|
|
81
|
+
].filter((s, i, arr) => arr.indexOf(s) === i).slice(0, 5),
|
|
82
|
+
affectedTools: incompleteTools.map((t) => t.tool),
|
|
83
|
+
});
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
export default LLM_004;
|
|
87
|
+
//# sourceMappingURL=LLM-004.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LLM-004.js","sourceRoot":"","sources":["../../../src/rules/llm/LLM-004.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,MAAM,CAAC,MAAM,OAAO,GAAS;IAC3B,EAAE,EAAE,SAAS;IACb,QAAQ,EAAE,aAAa;IACvB,IAAI,EAAE,0BAA0B;IAChC,WAAW,EAAE,8DAA8D;IAC3E,QAAQ,EAAE,SAAS;IACnB,QAAQ,EAAE,EAAE;IAEZ,KAAK,CAAC,KAAK,CAAC,OAAoB;QAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAE5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,gBAAgB,CAAC,SAAS,EAAE,EAAE,EAAE;gBACrC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,EAAE;gBACT,OAAO,EAAE,yCAAyC;aACnD,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,mBAAmB,EAAE,CAAC;QAExC,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,CAAC;YAChC,OAAO,gBAAgB,CAAC,SAAS,EAAE,EAAE,EAAE;gBACrC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,EAAE;gBACT,OAAO,EAAE,gDAAgD;gBACzD,OAAO,EAAE,CAAC,+CAA+C,CAAC;aAC3D,CAAC,CAAC;QACL,CAAC;QAED,MAAM,eAAe,GAKf,EAAE,CAAC;QACT,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;YAEzE,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBACvC,yBAAyB;gBACzB,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;gBACjD,eAAe,CAAC,IAAI,CAAC;oBACnB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,WAAW,EAAE,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBACvE,WAAW,EAAE,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC;oBACxE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;iBACtC,CAAC,CAAC;YACL,CAAC;YAED,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC;QAC7B,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAEzD,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,gBAAgB,CAAC,SAAS,EAAE,EAAE,EAAE;gBACrC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE,oCAAoC;aAC9C,CAAC,CAAC;QACL,CAAC;QAED,OAAO,gBAAgB,CAAC,SAAS,EAAE,EAAE,EAAE;YACrC,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,eAAe;YACtB,OAAO,EAAE,GAAG,eAAe,CAAC,MAAM,uCAAuC;YACzE,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACjC,MAAM,OAAO,GAAa,EAAE,CAAC;gBAC7B,IAAI,CAAC,CAAC,WAAW;oBAAE,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAChD,IAAI,CAAC,CAAC,WAAW;oBAAE,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBAClD,OAAO,SAAS,CAAC,CAAC,IAAI,iBAAiB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,SAAS,EAAE,CAAC;YAC9E,CAAC,CAAC;YACF,WAAW,EAAE;gBACX,sDAAsD;gBACtD,0CAA0C;gBAC1C,uGAAuG;gBACvG,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;aACjD,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YACzD,aAAa,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SAClD,CAAC,CAAC;IACL,CAAC;CACF,CAAC;AAEF,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-005: Full Quality Analysis
|
|
3
|
+
*
|
|
4
|
+
* Comprehensive LLM-based quality analysis of tool descriptions.
|
|
5
|
+
* This is the "premium" LLM rule that provides:
|
|
6
|
+
* - Overall quality score
|
|
7
|
+
* - Aspect-by-aspect breakdown
|
|
8
|
+
* - Suggested improved descriptions
|
|
9
|
+
* - Priority improvements
|
|
10
|
+
*
|
|
11
|
+
* This rule has a higher weight as it represents the most
|
|
12
|
+
* thorough AI-powered analysis.
|
|
13
|
+
*/
|
|
14
|
+
import type { Rule } from '../../core/types/rule.js';
|
|
15
|
+
export declare const LLM_005: Rule;
|
|
16
|
+
export default LLM_005;
|