rubrkit 0.1.0

package/src/localChecks.js

@@ -0,0 +1,680 @@
+// @ts-check
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import { usageError } from './errors.js';
+import { hashContent } from './manifest.js';
+
+const DEFAULT_MAX_TEXT_BYTES = 2 * 1024 * 1024;
+const IGNORED_DIRS = new Set(['.git', '.next', 'node_modules']);
+const TEXT_EXTENSIONS = new Set([
+    '.md',
+    '.mdx',
+    '.txt',
+    '.rubr',
+    '.rubr_flow',
+    '.json',
+    '.yaml',
+    '.yml',
+    '.js',
+    '.jsx',
+    '.ts',
+    '.tsx',
+]);
+const RUBR_FLOW_EXTENSIONS = new Set(['.rubr', '.rubr_flow']);
+const REQUIRED_RUBR_FLOW_BLOCKS = ['TASK', 'FLOW', 'OUTPUT'];
+const VAGUE_PATTERN = /\b(as needed|if useful|if needed|appropriate|better|good|etc\.?)\b/i;
+
+/**
+ * @param {{
+ *   target: string | null,
+ *   cwd?: string,
+ *   command?: string,
+ *   failUnder?: number | null,
+ *   failOn?: string | null,
+ *   fsImpl?: typeof fs
+ * }} params
+ */
+export async function runLocalChecks({ target, cwd = process.cwd(), command = 'validate', failUnder = null, failOn = null, fsImpl = fs }) {
+    if (!target) {
+        throw usageError(`The ${command} command needs a path or glob target for local checks.`);
+    }
+
+    const discovered = await discoverTargets({ target, cwd, fsImpl });
+    const files = [];
+
+    for (const filePath of discovered) {
+        files.push(await checkFile({ filePath, cwd, fsImpl }));
+    }
+
+    const summary = summarizeFiles(files);
+    const gates = evaluateQualityGates({ summary, files, failUnder, failOn });
+
+    return {
+        schemaVersion: 1,
+        command,
+        mode: 'local',
+        target,
+        generatedAt: new Date().toISOString(),
+        passed: summary.errorCount === 0 && gates.passed,
+        summary: {
+            ...summary,
+            score: gates.score,
+        },
+        gates,
+        files,
+    };
+}
+
+/**
+ * @param {{ target: string, cwd: string, fsImpl: typeof fs }} params
+ */
+export async function discoverTargets({ target, cwd, fsImpl }) {
+    if (hasGlob(target)) {
+        const matches = await expandGlob({ pattern: target, cwd, fsImpl });
+        if (matches.length === 0) {
+            throw usageError(`No files matched "${target}".`);
+        }
+
+        return matches;
+    }
+
+    const resolved = path.resolve(cwd, target);
+    let stats;
+
+    try {
+        stats = await fsImpl.stat(resolved);
+    } catch (error) {
+        throw usageError(`Could not read target "${target}": ${error instanceof Error ? error.message : 'not found'}`);
+    }
+
+    if (stats.isDirectory()) {
+        const files = await listTextFiles(resolved, fsImpl);
+        if (files.length === 0) {
+            throw usageError(`No supported text artifacts found under "${target}".`);
+        }
+
+        return files;
+    }
+
+    if (!stats.isFile()) {
+        throw usageError(`Target "${target}" is not a file or directory.`);
+    }
+
+    return [resolved];
+}
+
+/**
+ * @param {{
+ *   target: string,
+ *   cwd?: string,
+ *   fsImpl?: typeof fs,
+ *   message?: string | null,
+ * }} params
+ */
+export async function collectArtifactFilesForUpload({ target, cwd = process.cwd(), fsImpl = fs, message = null }) {
+    const discovered = await discoverTargets({ target, cwd, fsImpl });
+    const files = [];
+
+    for (const filePath of discovered) {
+        const relativePath = slash(path.relative(cwd, filePath) || path.basename(filePath));
+        let content;
+
+        try {
+            content = await fsImpl.readFile(filePath, 'utf8');
+        } catch (error) {
+            throw usageError(`Could not read target "${relativePath}": ${error instanceof Error ? error.message : 'not found'}`);
+        }
+
+        files.push({
+            path: relativePath,
+            content,
+            mediaType: mediaTypeForPath(filePath),
+            artifactType: inferArtifactType(filePath),
+            message,
+        });
+    }
+
+    return files;
+}
+
+/**
+ * @param {{ filePath: string, cwd: string, fsImpl: typeof fs }} params
+ */
+async function checkFile({ filePath, cwd, fsImpl }) {
+    const relativePath = slash(path.relative(cwd, filePath) || path.basename(filePath));
+    const issues = [];
+    const extension = getExtension(filePath);
+    const stats = await fsImpl.stat(filePath);
+    const artifactType = inferArtifactType(filePath);
+    let content = '';
+
+    if (stats.size > DEFAULT_MAX_TEXT_BYTES) {
+        issues.push(createIssue({
+            severity: 'error',
+            level: 'high',
+            code: 'file_too_large',
+            message: `File is ${stats.size} bytes, above the ${DEFAULT_MAX_TEXT_BYTES} byte local-check limit.`,
+            fix: 'Split the artifact or run a remote Rubrkit audit against a stored artifact bundle.',
+        }));
+
+        return finalizeFile({ relativePath, stats, artifactType, contentHash: null, issues });
+    }
+
+    try {
+        content = await fsImpl.readFile(filePath, 'utf8');
+    } catch (error) {
+        issues.push(createIssue({
+            severity: 'error',
+            level: 'high',
+            code: 'file_read_failed',
+            message: error instanceof Error ? error.message : 'Could not read file.',
+            fix: 'Confirm the artifact is a readable UTF-8 text file.',
+        }));
+        return finalizeFile({ relativePath, stats, artifactType, contentHash: null, issues });
+    }
+
+    if (!content.trim()) {
+        issues.push(createIssue({
+            severity: 'error',
+            level: 'high',
+            code: 'empty_artifact',
+            message: 'Artifact is empty.',
+            fix: 'Add the instruction artifact content before testing it.',
+        }));
+    }
+
+    if (!TEXT_EXTENSIONS.has(extension)) {
+        issues.push(createIssue({
+            severity: 'warning',
+            level: 'medium',
+            code: 'unknown_text_type',
+            message: 'Artifact extension is not one of Rubrkit local validation known text types.',
+            fix: 'Use a known text extension or run a remote audit after uploading the artifact bundle.',
+        }));
+    }
+
+    if (artifactType === 'rubr_flow' || looksLikeRubrFlow(content)) {
+        issues.push(...validateRubrFlowStructure(content, relativePath));
+    } else {
+        issues.push(...runGenericInstructionChecks(content));
+    }
+
+    return finalizeFile({ relativePath, stats, artifactType, contentHash: hashContent(content), issues });
+}
+
+/**
+ * @param {{
+ *   relativePath: string,
+ *   stats: import('node:fs').Stats,
+ *   artifactType: string,
+ *   contentHash: string | null,
+ *   issues: Array<Record<string, any>>
+ * }} params
+ */
+function finalizeFile({ relativePath, stats, artifactType, contentHash, issues }) {
+    const errorCount = issues.filter(issue => issue.severity === 'error').length;
+    const warningCount = issues.filter(issue => issue.severity === 'warning').length;
+
+    return {
+        path: relativePath,
+        artifactType,
+        sizeBytes: stats.size,
+        contentHash,
+        passed: errorCount === 0,
+        errorCount,
+        warningCount,
+        issues,
+    };
+}
+
+/**
+ * @param {string} source
+ * @param {string} sourceName
+ */
+function validateRubrFlowStructure(source, sourceName) {
+    const normalized = normalizeRubrFlowSource(source);
+    const lines = normalized.split('\n');
+    const issues = [];
+    const blocks = new Map();
+    let firstMeaningfulLine = 0;
+    let currentBlock = '';
+    let flowChildren = 0;
+    let outputChildren = 0;
+    let hasStopOrPass = false;
+
+    for (let index = 0; index < lines.length; index += 1) {
+        const lineNumber = index + 1;
+        const raw = lines[index];
+        const trimmed = raw.trim();
+
+        if (!trimmed || trimmed.startsWith('#')) {
+            continue;
+        }
+
+        const indent = raw.match(/^ */)?.[0].length ?? 0;
+        const token = trimmed.match(/^[^\s]+/)?.[0] ?? '';
+        const upperToken = token.toUpperCase();
+
+        if (!firstMeaningfulLine) {
+            firstMeaningfulLine = lineNumber;
+            if (upperToken !== 'TASK') {
+                issues.push(createIssue({
+                    severity: 'error',
+                    level: 'high',
+                    code: 'task_must_be_first',
+                    message: `${sourceName} should start with TASK.`,
+                    fix: 'Move the objective to the first meaningful line as TASK "Short imperative name".',
+                    line: lineNumber,
+                }));
+            }
+        }
+
+        if (raw.includes('\t')) {
+            issues.push(createIssue({
+                severity: 'error',
+                level: 'high',
+                code: 'tabs_not_allowed',
+                message: 'Indentation should use spaces, not tabs.',
+                fix: 'Replace tabs with 2-space indentation.',
+                line: lineNumber,
+            }));
+        }
+
+        if (indent % 2 !== 0) {
+            issues.push(createIssue({
+                severity: 'error',
+                level: 'high',
+                code: 'invalid_indentation',
+                message: 'Nested statements should use 2-space indentation.',
+                fix: 'Adjust the leading spaces to a multiple of 2.',
+                line: lineNumber,
+            }));
+        }
+
+        if (indent === 0) {
+            currentBlock = upperToken;
+            if (REQUIRED_RUBR_FLOW_BLOCKS.includes(upperToken) || ['VERIFY', 'RULES', 'CONTEXT', 'INPUTS', 'TOOLS', 'STATE'].includes(upperToken)) {
+                blocks.set(upperToken, lineNumber);
+            }
+
+            if (upperToken === 'TASK' && !/^TASK\s+"[^"]+"\s*$/.test(trimmed)) {
+                issues.push(createIssue({
+                    severity: 'error',
+                    level: 'high',
+                    code: 'invalid_task_statement',
+                    message: 'TASK should be a single quoted objective.',
+                    fix: 'Use a statement like TASK "Review release instructions".',
+                    line: lineNumber,
+                }));
+            }
+
+            continue;
+        }
+
+        if (currentBlock === 'FLOW') {
+            flowChildren += 1;
+        }
+
+        if (currentBlock === 'OUTPUT') {
+            outputChildren += 1;
+        }
+
+        if (/^(STOP|PASS)\b/.test(trimmed)) {
+            hasStopOrPass = true;
+        }
+    }
+
+    for (const block of REQUIRED_RUBR_FLOW_BLOCKS) {
+        if (!blocks.has(block)) {
+            issues.push(createIssue({
+                severity: 'error',
+                level: 'high',
+                code: 'missing_required_block',
+                message: `Missing required ${block} block.`,
+                fix: `Add ${block}${block === 'TASK' ? ' "Short imperative name"' : ''}.`,
+                line: firstMeaningfulLine || 1,
+            }));
+        }
+    }
+
+    if (blocks.has('FLOW') && flowChildren === 0) {
+        issues.push(createIssue({
+            severity: 'error',
+            level: 'high',
+            code: 'empty_flow_block',
+            message: 'FLOW is present but empty.',
+            fix: 'Add indented executable steps under FLOW.',
+            line: blocks.get('FLOW') ?? 1,
+        }));
+    }
+
+    if (blocks.has('OUTPUT') && outputChildren === 0) {
+        issues.push(createIssue({
+            severity: 'error',
+            level: 'high',
+            code: 'empty_output_block',
+            message: 'OUTPUT is present but empty.',
+            fix: 'Add named output fields under OUTPUT.',
+            line: blocks.get('OUTPUT') ?? 1,
+        }));
+    }
+
+    if (!blocks.has('VERIFY')) {
+        issues.push(createIssue({
+            severity: 'warning',
+            level: 'medium',
+            code: 'missing_verify_block',
+            message: 'Missing recommended VERIFY block.',
+            fix: 'Add checks that prove the procedure is complete.',
+            line: firstMeaningfulLine || 1,
+        }));
+    }
+
+    if (!hasStopOrPass) {
+        issues.push(createIssue({
+            severity: 'warning',
+            level: 'medium',
+            code: 'missing_stop_condition',
+            message: 'No explicit STOP or PASS condition was found.',
+            fix: 'Add STOP WHEN or PASS WHEN so an agent knows when the procedure is done.',
+            line: firstMeaningfulLine || 1,
+        }));
+    }
+
+    return issues;
+}
+
+/**
+ * @param {string} source
+ */
+function runGenericInstructionChecks(source) {
+    const issues = [];
+    const lower = source.toLowerCase();
+
+    if (!/\b(output|respond|return|deliverable|format)\b/.test(lower)) {
+        issues.push(createIssue({
+            severity: 'warning',
+            level: 'medium',
+            code: 'missing_output_contract',
+            message: 'Artifact does not appear to declare an output contract.',
+            fix: 'Name the expected output, format, and required fields.',
+        }));
+    }
+
+    if (!/\b(verify|test|check|acceptance|done when|pass)\b/.test(lower)) {
+        issues.push(createIssue({
+            severity: 'warning',
+            level: 'medium',
+            code: 'missing_verification',
+            message: 'Artifact does not appear to include verification or acceptance criteria.',
+            fix: 'Add concrete checks that prove the instruction was followed.',
+        }));
+    }
+
+    if (VAGUE_PATTERN.test(source)) {
+        issues.push(createIssue({
+            severity: 'warning',
+            level: 'medium',
+            code: 'vague_instruction_language',
+            message: 'Artifact contains language that can weaken repeatability.',
+            fix: 'Replace subjective or open-ended phrasing with observable conditions.',
+        }));
+    }
+
+    return issues;
+}
+
+/**
+ * @param {string} source
+ */
+function normalizeRubrFlowSource(source) {
+    const withoutBom = source.replace(/^\uFEFF/, '');
+    const trimmed = withoutBom.trim();
+    const fenced = trimmed.match(/^```(?:rubr_flow|rubr-flow|text|txt)?\s*\n([\s\S]*?)\n```$/i);
+
+    return (fenced?.[1] ?? withoutBom).replace(/\r\n?/g, '\n');
+}
+
+/**
+ * @param {string} source
+ */
+function looksLikeRubrFlow(source) {
+    const normalized = normalizeRubrFlowSource(source);
+    return /^\s*TASK\b/m.test(normalized) && /^\s*FLOW\b/m.test(normalized);
+}
+
+/**
+ * @param {{ severity: 'error' | 'warning', level: 'high' | 'medium' | 'low', code: string, message: string, fix: string, line?: number, column?: number }} issue
+ */
+function createIssue({ severity, level, code, message, fix, line = null, column = null }) {
+    return { severity, level, code, message, fix, line, column };
+}
+
+/**
+ * @param {Array<Record<string, any>>} files
+ */
+function summarizeFiles(files) {
+    const errorCount = files.reduce((sum, file) => sum + file.errorCount, 0);
+    const warningCount = files.reduce((sum, file) => sum + file.warningCount, 0);
+
+    return {
+        fileCount: files.length,
+        passedFiles: files.filter(file => file.passed).length,
+        failedFiles: files.filter(file => !file.passed).length,
+        errorCount,
+        warningCount,
+    };
+}
+
+/**
+ * @param {{ summary: Record<string, number>, files: Array<Record<string, any>>, failUnder?: number | null, failOn?: string | null }} params
+ */
+function evaluateQualityGates({ summary, files, failUnder = null, failOn = null }) {
+    const score = Math.max(0, Math.min(100, 100 - summary.errorCount * 30 - summary.warningCount * 5));
+    const failures = [];
+
+    if (typeof failUnder === 'number' && score < failUnder) {
+        failures.push({
+            code: 'score_below_threshold',
+            message: `Score ${score} is below --fail-under ${failUnder}.`,
+        });
+    }
+
+    if (failOn) {
+        const matching = files.flatMap(file => file.issues).filter(issue => issueMatchesFailOn(issue, failOn));
+
+        if (matching.length > 0) {
+            failures.push({
+                code: 'fail_on_threshold',
+                message: `${matching.length} issue${matching.length === 1 ? '' : 's'} matched --fail-on ${failOn}.`,
+            });
+        }
+    }
+
+    return {
+        passed: failures.length === 0,
+        score,
+        failUnder,
+        failOn,
+        failures,
+    };
+}
+
+/**
+ * @param {Record<string, any>} issue
+ * @param {string} failOn
+ */
+function issueMatchesFailOn(issue, failOn) {
+    if (failOn === 'critical') {
+        return issue.level === 'critical';
+    }
+
+    if (failOn === 'high') {
+        return ['critical', 'high'].includes(issue.level);
+    }
+
+    if (failOn === 'medium') {
+        return ['critical', 'high', 'medium'].includes(issue.level);
+    }
+
+    return ['critical', 'high', 'medium', 'low'].includes(issue.level);
+}
+
+/**
+ * @param {{ pattern: string, cwd: string, fsImpl: typeof fs }} params
+ */
+async function expandGlob({ pattern, cwd, fsImpl }) {
+    const absolutePattern = slash(path.resolve(cwd, pattern));
+    const root = findStaticRoot(absolutePattern);
+    const regex = globToRegExp(absolutePattern);
+    const candidates = await listTextFiles(root, fsImpl);
+
+    return candidates.filter(candidate => regex.test(slash(candidate))).sort();
+}
+
+/**
+ * @param {string} root
+ * @param {typeof fs} fsImpl
+ */
+async function listTextFiles(root, fsImpl) {
+    const entries = await fsImpl.readdir(root, { withFileTypes: true });
+    const files = [];
+
+    for (const entry of entries) {
+        if (IGNORED_DIRS.has(entry.name)) {
+            continue;
+        }
+
+        const fullPath = path.join(root, entry.name);
+
+        if (entry.isDirectory()) {
+            files.push(...(await listTextFiles(fullPath, fsImpl)));
+            continue;
+        }
+
+        if (entry.isFile() && TEXT_EXTENSIONS.has(getExtension(fullPath))) {
+            files.push(fullPath);
+        }
+    }
+
+    return files.sort();
+}
+
+/**
+ * @param {string} pattern
+ */
+function hasGlob(pattern) {
+    return /[*?[\]]/.test(pattern);
+}
+
+/**
+ * @param {string} absolutePattern
+ */
+function findStaticRoot(absolutePattern) {
+    const marker = absolutePattern.search(/[*?[\]]/);
+    const prefix = marker === -1 ? absolutePattern : absolutePattern.slice(0, marker);
+    const normalized = prefix.endsWith('/') ? prefix.slice(0, -1) : prefix;
+    const root = normalized.includes('/') ? normalized.slice(0, normalized.lastIndexOf('/')) : normalized;
+
+    return path.resolve(root || '.');
+}
+
+/**
+ * @param {string} pattern
+ */
+function globToRegExp(pattern) {
+    let source = '';
+
+    for (let index = 0; index < pattern.length; index += 1) {
+        const char = pattern[index];
+        const next = pattern[index + 1];
+
+        if (char === '*' && next === '*') {
+            source += '.*';
+            index += 1;
+        } else if (char === '*') {
+            source += '[^/]*';
+        } else if (char === '?') {
+            source += '[^/]';
+        } else {
+            source += escapeRegExp(char);
+        }
+    }
+
+    return new RegExp(`^${source}$`);
+}
+
+/**
+ * @param {string} value
+ */
+function escapeRegExp(value) {
+    return value.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&');
+}
+
+/**
+ * @param {string} filePath
+ */
+function getExtension(filePath) {
+    if (filePath.toLowerCase().endsWith('.rubr_flow')) {
+        return '.rubr_flow';
+    }
+
+    return path.extname(filePath).toLowerCase();
+}
+
+/**
+ * @param {string} filePath
+ */
+function inferArtifactType(filePath) {
+    const ext = getExtension(filePath);
+
+    if (RUBR_FLOW_EXTENSIONS.has(ext)) {
+        return 'rubr_flow';
+    }
+
+    if (ext === '.md' || ext === '.mdx') {
+        return 'markdown';
+    }
+
+    if (ext === '.json') {
+        return 'json';
+    }
+
+    if (ext === '.yaml' || ext === '.yml') {
+        return 'yaml';
+    }
+
+    if (ext === '.js' || ext === '.jsx') {
+        return 'javascript';
+    }
+
+    if (ext === '.ts' || ext === '.tsx') {
+        return 'typescript';
+    }
+
+    return 'text';
+}
+
+/**
+ * @param {string} filePath
+ */
+function mediaTypeForPath(filePath) {
+    const ext = getExtension(filePath);
+
+    if (ext === '.md' || ext === '.mdx') return 'text/markdown';
+    if (ext === '.json') return 'application/json';
+    if (ext === '.yaml' || ext === '.yml') return 'application/yaml';
+    if (ext === '.js' || ext === '.jsx') return 'application/javascript';
+    if (ext === '.ts' || ext === '.tsx') return 'application/typescript';
+
+    return 'text/plain';
+}
+
+/**
+ * @param {string} value
+ */
+function slash(value) {
+    return value.replace(/\\/g, '/');
+}