tokrepo 3.7.0 → 3.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/tokrepo.js +400 -14
- package/package.json +1 -1
package/bin/tokrepo.js
CHANGED
|
@@ -25,7 +25,7 @@ const CONFIG_DIR = path.join(os.homedir(), '.tokrepo');
|
|
|
25
25
|
const CONFIG_FILE = path.join(CONFIG_DIR, 'config.json');
|
|
26
26
|
const PROJECT_CONFIG = '.tokrepo.json';
|
|
27
27
|
const DEFAULT_API = 'https://api.tokrepo.com';
|
|
28
|
-
const CLI_VERSION = '3.
|
|
28
|
+
const CLI_VERSION = '3.9.0';
|
|
29
29
|
const VERSION_CHECK_FILE = path.join(os.homedir(), '.tokrepo', '.version-check');
|
|
30
30
|
const CODEX_DIR = path.join(os.homedir(), '.codex');
|
|
31
31
|
const CODEX_SKILLS_DIR = path.join(CODEX_DIR, 'skills');
|
|
@@ -302,9 +302,9 @@ function parseArgs(argv) {
|
|
|
302
302
|
}
|
|
303
303
|
|
|
304
304
|
const valueFlags = new Set([
|
|
305
|
-
'title', 'desc', 'tag', 'target', 'targets', 'keyword', 'types',
|
|
305
|
+
'title', 'desc', 'tag', 'target', 'targets', 'keyword', 'query', 'types',
|
|
306
306
|
'kind', 'install-mode', 'install_mode', 'entrypoint', 'asset-kind', 'asset_kind',
|
|
307
|
-
'version',
|
|
307
|
+
'version', 'uuid',
|
|
308
308
|
'policy', 'session',
|
|
309
309
|
'page', 'page-size', 'page_size', 'sort-by', 'sort_by',
|
|
310
310
|
'time-window', 'time_window',
|
|
@@ -546,13 +546,152 @@ function browserAuthFlow() {
|
|
|
546
546
|
});
|
|
547
547
|
}
|
|
548
548
|
|
|
549
|
-
|
|
550
|
-
const
|
|
549
|
+
function inferAssetKindFromPushFiles(files = []) {
|
|
550
|
+
const normalized = files.map(file => ({
|
|
551
|
+
...file,
|
|
552
|
+
lowerName: String(file.name || '').toLowerCase(),
|
|
553
|
+
content: String(file.content || ''),
|
|
554
|
+
}));
|
|
555
|
+
if (normalized.some(file => /"mcpServers"\s*:/.test(file.content) || /\bmcpServers\s*:/.test(file.content))) return 'mcp_config';
|
|
556
|
+
if (normalized.some(file => file.type === 'script' || /\.(sh|py|js|mjs|ts|rb|go|rs|lua)$/.test(file.lowerName) || /^#!\//.test(file.content))) return 'script';
|
|
557
|
+
if (normalized.some(file => file.lowerName === 'package.json' && /"bin"\s*:/.test(file.content))) return 'cli_tool';
|
|
558
|
+
if (normalized.some(file => file.type === 'skill' || isCodexSkillDocument(file))) return 'skill';
|
|
559
|
+
if (normalized.some(file => file.type === 'prompt')) return 'prompt';
|
|
560
|
+
if (normalized.some(file => file.type === 'config')) return 'config';
|
|
561
|
+
if (normalized.some(file => file.lowerName.endsWith('.md'))) return 'knowledge';
|
|
562
|
+
return 'other';
|
|
563
|
+
}
|
|
551
564
|
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
565
|
+
function analyzePushMetadataQuality(files = [], metadata = {}) {
|
|
566
|
+
const issues = [];
|
|
567
|
+
const add = (severity, code, message, fix = '') => {
|
|
568
|
+
issues.push({ severity, code, message, fix });
|
|
569
|
+
};
|
|
570
|
+
|
|
571
|
+
const normalizedKind = normalizeToolName(metadata.kind || '');
|
|
572
|
+
const inferredKind = inferAssetKindFromPushFiles(files);
|
|
573
|
+
const targetTools = parseCsvList(metadata.targetTools).map(normalizeToolName);
|
|
574
|
+
const installMode = normalizeCodexInstallMode(metadata.installMode);
|
|
575
|
+
const rawInstallMode = String(metadata.installMode || '').trim();
|
|
576
|
+
const entrypoint = String(metadata.entrypoint || '').trim();
|
|
577
|
+
const fileNames = new Set(files.map(file => String(file.name || '').replace(/\\/g, '/')));
|
|
578
|
+
const lowerFileNames = new Set(Array.from(fileNames).map(name => name.toLowerCase()));
|
|
579
|
+
const multiFile = files.length > 1;
|
|
580
|
+
const riskFlags = Array.from(new Set(files.flatMap(file => analyzeInstallRisks(file.name, file.content, file.type))));
|
|
581
|
+
const containsCodex = targetTools.includes('codex');
|
|
582
|
+
|
|
583
|
+
if (!normalizedKind) {
|
|
584
|
+
add('warning', 'missing_asset_kind', `No asset_kind was provided; inferred "${inferredKind}".`, `Recommended: --kind ${inferredKind}`);
|
|
585
|
+
} else if (!['skill', 'prompt', 'knowledge', 'mcp_config', 'script', 'cli_tool', 'config', 'pack', 'other'].includes(normalizedKind)) {
|
|
586
|
+
add('warning', 'unknown_asset_kind', `Unknown asset_kind "${metadata.kind}".`, 'Recommended: use skill, prompt, knowledge, mcp_config, script, cli_tool, config, or pack.');
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
if (targetTools.length === 0) {
|
|
590
|
+
add('warning', 'missing_target_tools', 'No target_tools metadata was provided.', 'Recommended: --target codex or --targets codex,claude_code,gemini_cli.');
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
const invalidTargets = targetTools.filter(tool => !['codex', 'claude_code', 'gemini_cli', 'cursor', 'windsurf', 'opencode'].includes(tool));
|
|
594
|
+
if (invalidTargets.length > 0) {
|
|
595
|
+
add('info', 'unknown_target_tool', `Unknown target tool(s): ${invalidTargets.join(', ')}.`, 'Use stable ids when possible: codex, claude_code, gemini_cli, cursor, windsurf.');
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
if (rawInstallMode && !installMode) {
|
|
599
|
+
add('warning', 'unknown_install_mode', `Unknown install_mode "${metadata.installMode}".`, 'Recommended: single, bundle, split, or stage_only.');
|
|
600
|
+
} else if (!rawInstallMode && (multiFile || containsCodex)) {
|
|
601
|
+
add('warning', 'missing_install_mode', `No install_mode was provided for ${multiFile ? 'a multi-file asset' : 'a Codex-targeted asset'}.`, `Recommended: --install-mode ${multiFile ? 'bundle' : 'single'}.`);
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
if (entrypoint && !fileNames.has(entrypoint) && !lowerFileNames.has(entrypoint.toLowerCase())) {
|
|
605
|
+
add('warning', 'entrypoint_missing', `Entrypoint "${entrypoint}" is not included in the pushed files.`, 'Recommended: set --entrypoint to an included file, usually SKILL.md.');
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
if (!entrypoint && (installMode === 'single' || installMode === 'bundle' || containsCodex)) {
|
|
609
|
+
add('warning', 'missing_entrypoint', 'No entrypoint metadata was provided.', 'Recommended: --entrypoint SKILL.md or the main markdown file.');
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
const effectiveKind = normalizedKind || inferredKind;
|
|
613
|
+
if (containsCodex && ['script', 'cli_tool', 'mcp_config'].includes(effectiveKind) && installMode !== 'stage_only') {
|
|
614
|
+
add('warning', 'high_risk_codex_activation', `${effectiveKind} assets targeted at Codex should stage by default.`, 'Recommended: --install-mode stage_only or publish a markdown skill wrapper.');
|
|
615
|
+
}
|
|
616
|
+
if (riskFlags.includes('executable')) {
|
|
617
|
+
add('warning', 'executes_code_detected', 'Executable code was detected in the asset files.', 'Recommended: declare script/cli_tool or use stage_only for agent installs.');
|
|
618
|
+
}
|
|
619
|
+
if (riskFlags.includes('mcp')) {
|
|
620
|
+
add('warning', 'mcp_config_detected', 'MCP config content was detected.', 'Recommended: --kind mcp_config --install-mode stage_only.');
|
|
621
|
+
}
|
|
622
|
+
if (riskFlags.includes('env')) {
|
|
623
|
+
add('info', 'secret_or_env_mentions', 'Environment variable or secret-like words were mentioned.', 'Check this is documentation only and no real secret is included.');
|
|
624
|
+
}
|
|
625
|
+
if (riskFlags.includes('absolute-path')) {
|
|
626
|
+
add('warning', 'absolute_path_detected', 'Absolute local paths were detected.', 'Recommended: replace machine-specific paths with placeholders.');
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
if (containsCodex && effectiveKind === 'skill') {
|
|
630
|
+
const skillDocs = files.filter(file => isCodexSkillDocument(file));
|
|
631
|
+
if (installMode === 'split' && skillDocs.length !== files.length) {
|
|
632
|
+
add('warning', 'split_requires_skill_docs', 'install_mode=split works best when every markdown file has skill frontmatter.', 'Recommended: use bundle or add name/description frontmatter to each split skill.');
|
|
633
|
+
}
|
|
634
|
+
if (skillDocs.length === 0) {
|
|
635
|
+
add('warning', 'missing_codex_skill_frontmatter', 'No Codex skill frontmatter was found.', 'Recommended: add YAML frontmatter with name and description.');
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
const severityPenalty = issues.reduce((sum, issue) => {
|
|
640
|
+
if (issue.severity === 'warning') return sum + 10;
|
|
641
|
+
return sum + 2;
|
|
642
|
+
}, 0);
|
|
643
|
+
const score = Math.max(0, 100 - severityPenalty);
|
|
644
|
+
const status = issues.some(issue => issue.severity === 'warning') ? 'warn' : 'pass';
|
|
645
|
+
|
|
646
|
+
return {
|
|
647
|
+
score,
|
|
648
|
+
status,
|
|
649
|
+
inferredAssetKind: inferredKind,
|
|
650
|
+
assetKind: normalizedKind || '',
|
|
651
|
+
targetTools,
|
|
652
|
+
installMode: installMode || '',
|
|
653
|
+
entrypoint,
|
|
654
|
+
riskFlags,
|
|
655
|
+
issueCount: issues.length,
|
|
656
|
+
issues,
|
|
657
|
+
};
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
function formatMetadataQualityLabel(report) {
|
|
661
|
+
const color = report.status === 'pass' ? C.green : C.yellow;
|
|
662
|
+
return `${color}${report.status}${C.reset} ${C.bold}${report.score}/100${C.reset}`;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
function printMetadataQualityReport(report, opts = {}) {
|
|
666
|
+
const compact = Boolean(opts.compact);
|
|
667
|
+
if (!compact) {
|
|
668
|
+
log(`\n${C.bold}Agent metadata quality${C.reset}`);
|
|
669
|
+
log(` Status: ${formatMetadataQualityLabel(report)}`);
|
|
670
|
+
log(` Inferred kind: ${report.inferredAssetKind}`);
|
|
671
|
+
if (report.targetTools.length) log(` Targets: ${report.targetTools.join(', ')}`);
|
|
672
|
+
if (report.installMode) log(` Install mode: ${report.installMode}`);
|
|
673
|
+
if (report.riskFlags.length) log(` Risk flags: ${report.riskFlags.join(', ')}`);
|
|
674
|
+
} else {
|
|
675
|
+
log(`${C.bold}Agent metadata quality suggestions:${C.reset}`);
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
if (report.issues.length === 0) {
|
|
679
|
+
if (!compact) success('Metadata is agent-ready.');
|
|
680
|
+
return;
|
|
681
|
+
}
|
|
682
|
+
for (const issue of report.issues.slice(0, compact ? 8 : report.issues.length)) {
|
|
683
|
+
const color = issue.severity === 'warning' ? C.yellow : C.dim;
|
|
684
|
+
log(` ${color}${issue.severity.toUpperCase()}${C.reset} ${issue.code}: ${issue.message}`);
|
|
685
|
+
if (issue.fix) log(` ${C.dim}${issue.fix}${C.reset}`);
|
|
555
686
|
}
|
|
687
|
+
if (compact && report.issues.length > 8) {
|
|
688
|
+
log(` ${C.dim}...and ${report.issues.length - 8} more suggestion(s). Run --metadata-report for full detail.${C.reset}`);
|
|
689
|
+
}
|
|
690
|
+
if (!compact) log('');
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
async function cmdPush() {
|
|
694
|
+
const args = parseArgs(process.argv);
|
|
556
695
|
|
|
557
696
|
const projectConfig = readProjectConfig();
|
|
558
697
|
const baseDir = process.cwd();
|
|
@@ -623,6 +762,31 @@ async function cmdPush() {
|
|
|
623
762
|
error('No readable text files found to push.');
|
|
624
763
|
}
|
|
625
764
|
|
|
765
|
+
const metadataQuality = analyzePushMetadataQuality(pushFiles, {
|
|
766
|
+
kind,
|
|
767
|
+
targetTools,
|
|
768
|
+
installMode,
|
|
769
|
+
entrypoint,
|
|
770
|
+
title,
|
|
771
|
+
description,
|
|
772
|
+
});
|
|
773
|
+
|
|
774
|
+
if (args.flags.metadata_report || args.flags.metadataReport) {
|
|
775
|
+
if (args.flags.json) {
|
|
776
|
+
outputJson({ schemaVersion: 1, metadataQuality, files: pushFiles.map(file => ({ name: file.name, type: file.type, bytes: Buffer.byteLength(file.content || '') })) });
|
|
777
|
+
} else {
|
|
778
|
+
printMetadataQualityReport(metadataQuality);
|
|
779
|
+
}
|
|
780
|
+
return;
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
// Public registry policy: quality gates advise by default and never block user uploads.
|
|
784
|
+
// TokRepo-owned CI can opt into strict mode with TOKREPO_METADATA_STRICT=1.
|
|
785
|
+
if (process.env.TOKREPO_METADATA_STRICT === '1' && metadataQuality.issues.length > 0 && !args.flags.force) {
|
|
786
|
+
if (!args.flags.json) printMetadataQualityReport(metadataQuality);
|
|
787
|
+
error(`Internal metadata quality gate failed. Fix the issues or re-run with --force to push anyway.`);
|
|
788
|
+
}
|
|
789
|
+
|
|
626
790
|
// Show summary
|
|
627
791
|
log(`\n${C.bold}tokrepo push${C.reset}\n`);
|
|
628
792
|
log(` ${C.bold}Title:${C.reset} ${title}`);
|
|
@@ -640,6 +804,7 @@ async function cmdPush() {
|
|
|
640
804
|
if (metadataSummary.length > 0) {
|
|
641
805
|
log(` ${C.bold}Agent meta:${C.reset} ${metadataSummary.join(' · ')}`);
|
|
642
806
|
}
|
|
807
|
+
log(` ${C.bold}Agent quality:${C.reset} ${formatMetadataQualityLabel(metadataQuality)}`);
|
|
643
808
|
log('');
|
|
644
809
|
|
|
645
810
|
for (const f of pushFiles) {
|
|
@@ -650,9 +815,18 @@ async function cmdPush() {
|
|
|
650
815
|
|
|
651
816
|
const totalChars = pushFiles.reduce((sum, f) => sum + f.content.length, 0);
|
|
652
817
|
|
|
818
|
+
if (metadataQuality.issues.length > 0) {
|
|
819
|
+
printMetadataQualityReport(metadataQuality, { compact: true });
|
|
820
|
+
}
|
|
821
|
+
|
|
653
822
|
// Push
|
|
654
823
|
info('Pushing...');
|
|
655
824
|
|
|
825
|
+
const config = readConfig();
|
|
826
|
+
if (!config || !config.token) {
|
|
827
|
+
error(`Not logged in. Run: ${C.cyan}tokrepo login${C.reset}`);
|
|
828
|
+
}
|
|
829
|
+
|
|
656
830
|
try {
|
|
657
831
|
const data = await apiRequest('POST', '/api/v1/tokenboard/push/upsert', {
|
|
658
832
|
title,
|
|
@@ -825,17 +999,28 @@ async function cmdSearch() {
|
|
|
825
999
|
const apiBase = config?.api || DEFAULT_API;
|
|
826
1000
|
|
|
827
1001
|
try {
|
|
828
|
-
const encoded = encodeURIComponent(query);
|
|
829
1002
|
const pageSize = Number(args.flags.pageSize || (args.flags.all ? 200 : 20)) || 20;
|
|
830
1003
|
const sortBy = args.flags.sortBy || 'views';
|
|
831
1004
|
let page = Number(args.flags.page || 1) || 1;
|
|
832
|
-
|
|
1005
|
+
const buildSearchPath = (pageNo) => {
|
|
1006
|
+
const params = new URLSearchParams({
|
|
1007
|
+
keyword: query,
|
|
1008
|
+
page: String(pageNo),
|
|
1009
|
+
page_size: String(pageSize),
|
|
1010
|
+
sort_by: sortBy,
|
|
1011
|
+
});
|
|
1012
|
+
if (args.flags.target) params.set('target', args.flags.target);
|
|
1013
|
+
if (args.flags.kind || args.flags.assetKind) params.set('kind', args.flags.kind || args.flags.assetKind);
|
|
1014
|
+
if (args.flags.policy) params.set('policy', args.flags.policy);
|
|
1015
|
+
return `/api/v1/tokenboard/workflows/list?${params.toString()}`;
|
|
1016
|
+
};
|
|
1017
|
+
let data = await apiRequest('GET', buildSearchPath(page), null, config?.token, apiBase);
|
|
833
1018
|
|
|
834
1019
|
if (args.flags.all) {
|
|
835
1020
|
const list = [...(data.list || [])];
|
|
836
1021
|
while (list.length < (data.total || 0)) {
|
|
837
1022
|
page++;
|
|
838
|
-
const next = await apiRequest('GET',
|
|
1023
|
+
const next = await apiRequest('GET', buildSearchPath(page), null, config?.token, apiBase);
|
|
839
1024
|
const items = next.list || [];
|
|
840
1025
|
if (items.length === 0) break;
|
|
841
1026
|
list.push(...items);
|
|
@@ -892,9 +1077,12 @@ async function cmdSearch() {
|
|
|
892
1077
|
log(` ${C.dim}${String(i + 1).padStart(2)}.${C.reset} ${C.bold}${wf.title}${C.reset}`);
|
|
893
1078
|
if (desc) log(` ${desc}`);
|
|
894
1079
|
if (tags) log(` ${C.cyan}${tags}${C.reset} ${C.dim}★${votes} 👁${views}${C.reset}`);
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
1080
|
+
const fit = wf.agent_fit || wf.agentFit || wf.compatibility?.codex;
|
|
1081
|
+
if (fit) {
|
|
1082
|
+
const policy = fit.policy || fit.policyDecision?.decision || 'unknown';
|
|
1083
|
+
const kind = fit.asset_kind || fit.assetKind || 'unknown';
|
|
1084
|
+
const score = fit.score !== undefined ? ` · score=${fit.score}` : '';
|
|
1085
|
+
log(` ${C.dim}codex: ${fit.status || 'unknown'} · policy=${policy} · kind=${kind}${score}${C.reset}`);
|
|
898
1086
|
}
|
|
899
1087
|
log(` ${C.dim}tokrepo install ${wf.uuid}${C.reset}`);
|
|
900
1088
|
log('');
|
|
@@ -1582,6 +1770,22 @@ function publicInstallPlan(plan) {
|
|
|
1582
1770
|
}
|
|
1583
1771
|
|
|
1584
1772
|
function workflowCodexCompatibility(workflow) {
|
|
1773
|
+
if (workflow?.agent_fit || workflow?.agentFit) {
|
|
1774
|
+
const fit = workflow.agent_fit || workflow.agentFit;
|
|
1775
|
+
return {
|
|
1776
|
+
targetTool: fit.target || 'codex',
|
|
1777
|
+
status: fit.status || 'unknown',
|
|
1778
|
+
score: fit.score ?? 50,
|
|
1779
|
+
assetKind: fit.asset_kind || fit.assetKind || workflowAssetKind(workflow),
|
|
1780
|
+
targetTools: workflowTargetTools(workflow),
|
|
1781
|
+
installMode: fit.install_mode || fit.installMode || 'single',
|
|
1782
|
+
policyDecision: {
|
|
1783
|
+
decision: fit.policy || fit.policyDecision?.decision || 'allow',
|
|
1784
|
+
requiresConfirmation: ['confirm'].includes(fit.policy || ''),
|
|
1785
|
+
reasons: fit.why || fit.reasons || [],
|
|
1786
|
+
},
|
|
1787
|
+
};
|
|
1788
|
+
}
|
|
1585
1789
|
const metadata = workflowAgentMetadata(workflow);
|
|
1586
1790
|
const assetKind = workflowAssetKind(workflow);
|
|
1587
1791
|
const targetTools = workflowTargetTools(workflow);
|
|
@@ -1653,10 +1857,20 @@ function workflowMatchesAgentFilters(workflow, flags = {}) {
|
|
|
1653
1857
|
|
|
1654
1858
|
function enrichWorkflowForAgent(workflow) {
|
|
1655
1859
|
const compatibility = workflowCodexCompatibility(workflow);
|
|
1860
|
+
const agentFit = workflow.agent_fit || workflow.agentFit || {
|
|
1861
|
+
target: 'codex',
|
|
1862
|
+
score: compatibility.score,
|
|
1863
|
+
status: compatibility.status,
|
|
1864
|
+
policy: compatibility.policyDecision.decision,
|
|
1865
|
+
why: compatibility.policyDecision.reasons,
|
|
1866
|
+
asset_kind: compatibility.assetKind,
|
|
1867
|
+
install_mode: compatibility.installMode,
|
|
1868
|
+
};
|
|
1656
1869
|
return {
|
|
1657
1870
|
...workflow,
|
|
1658
1871
|
assetKind: compatibility.assetKind,
|
|
1659
1872
|
targetTools: compatibility.targetTools,
|
|
1873
|
+
agent_fit: agentFit,
|
|
1660
1874
|
compatibility: {
|
|
1661
1875
|
codex: compatibility,
|
|
1662
1876
|
},
|
|
@@ -2793,6 +3007,151 @@ async function fetchServerCodexInstallPlan(uuid, config, apiBase) {
|
|
|
2793
3007
|
}
|
|
2794
3008
|
}
|
|
2795
3009
|
|
|
3010
|
+
function runSelfCliJson(cliArgs, opts = {}) {
|
|
3011
|
+
const childProcess = require('child_process');
|
|
3012
|
+
const stdout = childProcess.execFileSync(process.execPath, [__filename, ...cliArgs], {
|
|
3013
|
+
env: { ...process.env, ...(opts.env || {}), TOKREPO_NONINTERACTIVE: '1' },
|
|
3014
|
+
cwd: opts.cwd || process.cwd(),
|
|
3015
|
+
encoding: 'utf8',
|
|
3016
|
+
maxBuffer: 50 * 1024 * 1024,
|
|
3017
|
+
});
|
|
3018
|
+
return JSON.parse(stdout);
|
|
3019
|
+
}
|
|
3020
|
+
|
|
3021
|
+
function makeEvalResult(name, ok, details = {}) {
|
|
3022
|
+
return {
|
|
3023
|
+
name,
|
|
3024
|
+
ok: Boolean(ok),
|
|
3025
|
+
status: ok ? 'pass' : 'fail',
|
|
3026
|
+
...details,
|
|
3027
|
+
};
|
|
3028
|
+
}
|
|
3029
|
+
|
|
3030
|
+
function createTempDir(prefix) {
|
|
3031
|
+
return fs.mkdtempSync(path.join(os.tmpdir(), `${prefix}-`));
|
|
3032
|
+
}
|
|
3033
|
+
|
|
3034
|
+
async function cmdEvalAgent() {
|
|
3035
|
+
const args = parseArgs(process.argv);
|
|
3036
|
+
const json = Boolean(args.flags.json);
|
|
3037
|
+
const sampleUuid = args.flags.uuid || '91aeb22d-eff0-4310-abc6-811d2394b420';
|
|
3038
|
+
const query = args.flags.keyword || args.flags.query || 'video';
|
|
3039
|
+
const keepTemp = Boolean(args.flags.keep_temp || args.flags.keepTemp);
|
|
3040
|
+
const startedAt = new Date().toISOString();
|
|
3041
|
+
const results = [];
|
|
3042
|
+
const tempRoots = [];
|
|
3043
|
+
|
|
3044
|
+
if (!json) log(`\n${C.bold}tokrepo eval-agent${C.reset}\n`);
|
|
3045
|
+
|
|
3046
|
+
const runScenario = async (name, fn) => {
|
|
3047
|
+
const start = Date.now();
|
|
3048
|
+
try {
|
|
3049
|
+
const details = await fn();
|
|
3050
|
+
const result = makeEvalResult(name, true, { durationMs: Date.now() - start, ...details });
|
|
3051
|
+
results.push(result);
|
|
3052
|
+
if (!json) success(`${name} (${result.durationMs}ms)`);
|
|
3053
|
+
} catch (e) {
|
|
3054
|
+
const result = makeEvalResult(name, false, { durationMs: Date.now() - start, error: e.message });
|
|
3055
|
+
results.push(result);
|
|
3056
|
+
if (!json) warn(`${name}: ${e.message}`);
|
|
3057
|
+
}
|
|
3058
|
+
};
|
|
3059
|
+
|
|
3060
|
+
await runScenario('search_filters_codex_allow_skill', async () => {
|
|
3061
|
+
const data = runSelfCliJson(['search', query, '--target', 'codex', '--kind', 'skill', '--policy', 'allow', '--json', '--page-size', '10']);
|
|
3062
|
+
if (!data.count || !Array.isArray(data.list)) throw new Error('filtered search returned no list');
|
|
3063
|
+
const bad = data.list.find(item => {
|
|
3064
|
+
const policy = item.agent_fit?.policy || item.policyDecision?.decision;
|
|
3065
|
+
return policy && policy !== 'allow';
|
|
3066
|
+
});
|
|
3067
|
+
if (bad) throw new Error(`search returned non-allow asset ${bad.uuid}`);
|
|
3068
|
+
const firstFit = data.list[0]?.agent_fit || data.list[0]?.agentFit;
|
|
3069
|
+
if (!firstFit?.score && firstFit?.score !== 0) throw new Error('search result missing agent_fit.score');
|
|
3070
|
+
if (firstFit.policy !== 'allow') throw new Error(`first result policy is ${firstFit.policy}`);
|
|
3071
|
+
return { count: data.count, firstUuid: data.list[0]?.uuid, firstTitle: data.list[0]?.title, firstAgentFitScore: firstFit.score };
|
|
3072
|
+
});
|
|
3073
|
+
|
|
3074
|
+
await runScenario('install_plan_contract', async () => {
|
|
3075
|
+
const plan = runSelfCliJson(['plan', sampleUuid, '--target', 'codex']);
|
|
3076
|
+
if (plan.schemaVersion !== 2) throw new Error(`expected schemaVersion 2, got ${plan.schemaVersion}`);
|
|
3077
|
+
if (!plan.policyDecision?.decision) throw new Error('missing policyDecision');
|
|
3078
|
+
if (!Array.isArray(plan.actions) || plan.actions.length === 0) throw new Error('missing actions');
|
|
3079
|
+
if (!Array.isArray(plan.rollback) || plan.rollback.length === 0) throw new Error('missing rollback');
|
|
3080
|
+
if (!Array.isArray(plan.postVerify) || plan.postVerify.length === 0) throw new Error('missing postVerify');
|
|
3081
|
+
return {
|
|
3082
|
+
sourceOfTruth: plan.sourceOfTruth,
|
|
3083
|
+
concretePlanSource: plan.concretePlanSource,
|
|
3084
|
+
policy: plan.policyDecision.decision,
|
|
3085
|
+
actions: plan.actions.length,
|
|
3086
|
+
};
|
|
3087
|
+
});
|
|
3088
|
+
|
|
3089
|
+
await runScenario('metadata_quality_report_non_blocking', async () => {
|
|
3090
|
+
const tmp = createTempDir('tokrepo-eval-quality');
|
|
3091
|
+
tempRoots.push(tmp);
|
|
3092
|
+
const skillPath = path.join(tmp, 'SKILL.md');
|
|
3093
|
+
fs.writeFileSync(skillPath, `---\nname: eval-agent-sample\ndescription: \"Sample skill used by tokrepo eval-agent.\"\n---\n\n# Eval Agent Sample\n\nUse this to test metadata quality reporting.\n`);
|
|
3094
|
+
const report = runSelfCliJson(['push', skillPath, '--metadata-report', '--json', '--kind', 'skill', '--target', 'codex', '--install-mode', 'single', '--entrypoint', 'SKILL.md']);
|
|
3095
|
+
if (!report.metadataQuality) throw new Error('missing metadataQuality');
|
|
3096
|
+
if (report.metadataQuality.status !== 'pass') throw new Error(`expected pass, got ${report.metadataQuality.status}`);
|
|
3097
|
+
return { score: report.metadataQuality.score, status: report.metadataQuality.status };
|
|
3098
|
+
});
|
|
3099
|
+
|
|
3100
|
+
await runScenario('codex_install_verify_and_rollback', async () => {
|
|
3101
|
+
const tmpHome = createTempDir('tokrepo-eval-home');
|
|
3102
|
+
tempRoots.push(tmpHome);
|
|
3103
|
+
const env = { HOME: tmpHome };
|
|
3104
|
+
const install = runSelfCliJson(['install', sampleUuid, '--target', 'codex', '--yes', '--json'], { env });
|
|
3105
|
+
if (!install.sessionId) throw new Error('install did not create sessionId');
|
|
3106
|
+
if (!install.verification?.ok) throw new Error('install verification failed');
|
|
3107
|
+
if (!install.installedFiles?.length) throw new Error('no files installed');
|
|
3108
|
+
const installed = runSelfCliJson(['installed', '--target', 'codex', '--json'], { env });
|
|
3109
|
+
if (installed.count < 1) throw new Error('installed manifest did not record asset');
|
|
3110
|
+
const rollback = runSelfCliJson(['rollback', '--last', '--target', 'codex', '--json'], { env });
|
|
3111
|
+
if (!rollback.removedFiles?.length) throw new Error('rollback removed no files');
|
|
3112
|
+
const after = runSelfCliJson(['installed', '--target', 'codex', '--json'], { env });
|
|
3113
|
+
if (after.count !== 0) throw new Error(`manifest still has ${after.count} install(s) after rollback`);
|
|
3114
|
+
return {
|
|
3115
|
+
installedFiles: install.installedFiles.length,
|
|
3116
|
+
sessionId: install.sessionId,
|
|
3117
|
+
removedFiles: rollback.removedFiles.length,
|
|
3118
|
+
};
|
|
3119
|
+
});
|
|
3120
|
+
|
|
3121
|
+
if (!keepTemp) {
|
|
3122
|
+
for (const dir of tempRoots) {
|
|
3123
|
+
try { fs.rmSync(dir, { recursive: true, force: true }); } catch {}
|
|
3124
|
+
}
|
|
3125
|
+
}
|
|
3126
|
+
|
|
3127
|
+
const failed = results.filter(result => !result.ok);
|
|
3128
|
+
const summary = {
|
|
3129
|
+
schemaVersion: 1,
|
|
3130
|
+
startedAt,
|
|
3131
|
+
finishedAt: new Date().toISOString(),
|
|
3132
|
+
cliVersion: CLI_VERSION,
|
|
3133
|
+
targetTool: 'codex',
|
|
3134
|
+
sampleUuid,
|
|
3135
|
+
query,
|
|
3136
|
+
status: failed.length === 0 ? 'pass' : 'fail',
|
|
3137
|
+
passed: results.length - failed.length,
|
|
3138
|
+
failed: failed.length,
|
|
3139
|
+
count: results.length,
|
|
3140
|
+
tempRoots: keepTemp ? tempRoots : [],
|
|
3141
|
+
results,
|
|
3142
|
+
};
|
|
3143
|
+
|
|
3144
|
+
if (json) {
|
|
3145
|
+
outputJson(summary);
|
|
3146
|
+
} else {
|
|
3147
|
+
log('');
|
|
3148
|
+
if (summary.status === 'pass') success(`Agent eval passed: ${summary.passed}/${summary.count}`);
|
|
3149
|
+
else warn(`Agent eval failed: ${summary.failed}/${summary.count}`);
|
|
3150
|
+
}
|
|
3151
|
+
|
|
3152
|
+
if (failed.length > 0) process.exitCode = 1;
|
|
3153
|
+
}
|
|
3154
|
+
|
|
2796
3155
|
async function cmdSyncInstalled() {
|
|
2797
3156
|
const args = parseArgs(process.argv);
|
|
2798
3157
|
const targetTool = validateInstallTarget(args.flags.target || 'codex');
|
|
@@ -3439,6 +3798,7 @@ ${C.bold}DISCOVER & INSTALL${C.reset}
|
|
|
3439
3798
|
${C.cyan}sync-installed${C.reset} Update installed Codex assets from manifest
|
|
3440
3799
|
${C.cyan}uninstall${C.reset} <uuid> Remove a managed Codex install
|
|
3441
3800
|
${C.cyan}rollback${C.reset} --last Roll back the latest Codex install session
|
|
3801
|
+
${C.cyan}eval-agent${C.reset} Run agent-native contract and lifecycle evals
|
|
3442
3802
|
|
|
3443
3803
|
${C.bold}PUBLISH${C.reset}
|
|
3444
3804
|
${C.cyan}push${C.reset} [files...] Push files/directory (idempotent upsert)
|
|
@@ -3464,6 +3824,7 @@ ${C.bold}PUSH OPTIONS${C.reset}
|
|
|
3464
3824
|
${C.cyan}--kind${C.reset} skill Set agent asset_kind
|
|
3465
3825
|
${C.cyan}--target${C.reset} codex Add target tool metadata on push
|
|
3466
3826
|
${C.cyan}--install-mode${C.reset} bundle Set install_mode metadata
|
|
3827
|
+
${C.cyan}--metadata-report${C.reset} Print agent metadata quality suggestions without pushing
|
|
3467
3828
|
|
|
3468
3829
|
${C.bold}INSTALL BEHAVIOR${C.reset}
|
|
3469
3830
|
Skills → .claude/skills/ (if .claude/ exists)
|
|
@@ -3489,7 +3850,9 @@ ${C.bold}EXAMPLES${C.reset}
|
|
|
3489
3850
|
tokrepo sync-installed --target codex --dry-run
|
|
3490
3851
|
tokrepo uninstall 91aeb22d --target codex --dry-run
|
|
3491
3852
|
tokrepo rollback --last --target codex --dry-run
|
|
3853
|
+
tokrepo eval-agent --json
|
|
3492
3854
|
tokrepo push --private my-rules.md # Save one file privately
|
|
3855
|
+
tokrepo push . --metadata-report --json # Check agent metadata without uploading
|
|
3493
3856
|
tokrepo push . --kind skill --target codex --install-mode bundle
|
|
3494
3857
|
tokrepo push --public skill.md # Share one file publicly
|
|
3495
3858
|
tokrepo push --private . # Push current dir as private
|
|
@@ -3666,6 +4029,25 @@ EXAMPLES
|
|
|
3666
4029
|
`);
|
|
3667
4030
|
}
|
|
3668
4031
|
|
|
4032
|
+
function showEvalAgentHelp() {
|
|
4033
|
+
log(`
|
|
4034
|
+
${C.bold}tokrepo eval-agent${C.reset}
|
|
4035
|
+
|
|
4036
|
+
USAGE
|
|
4037
|
+
tokrepo eval-agent [--json] [--uuid <asset-uuid>] [--keyword video] [--keep-temp]
|
|
4038
|
+
|
|
4039
|
+
BEHAVIOR
|
|
4040
|
+
Runs agent-native smoke evals against search filters, install-plan contracts,
|
|
4041
|
+
metadata quality reporting, Codex install verification, manifest state, and rollback.
|
|
4042
|
+
Lifecycle tests use a temporary HOME and do not touch your real ~/.codex.
|
|
4043
|
+
|
|
4044
|
+
EXAMPLES
|
|
4045
|
+
tokrepo eval-agent
|
|
4046
|
+
tokrepo eval-agent --json
|
|
4047
|
+
tokrepo eval-agent --uuid 91aeb22d-eff0-4310-abc6-811d2394b420 --keyword video --json
|
|
4048
|
+
`);
|
|
4049
|
+
}
|
|
4050
|
+
|
|
3669
4051
|
function showCommandHelp(command) {
|
|
3670
4052
|
switch (command) {
|
|
3671
4053
|
case 'search':
|
|
@@ -3693,6 +4075,9 @@ function showCommandHelp(command) {
|
|
|
3693
4075
|
showUninstallHelp(); break;
|
|
3694
4076
|
case 'rollback':
|
|
3695
4077
|
showRollbackHelp(); break;
|
|
4078
|
+
case 'eval-agent':
|
|
4079
|
+
case 'eval':
|
|
4080
|
+
showEvalAgentHelp(); break;
|
|
3696
4081
|
default:
|
|
3697
4082
|
showHelp(); break;
|
|
3698
4083
|
}
|
|
@@ -3725,6 +4110,7 @@ async function main() {
|
|
|
3725
4110
|
case 'installed': await cmdInstalled(); break;
|
|
3726
4111
|
case 'uninstall': case 'remove': case 'rm': await cmdUninstall(); break;
|
|
3727
4112
|
case 'rollback': await cmdRollback(); break;
|
|
4113
|
+
case 'eval-agent': case 'eval': await cmdEvalAgent(); break;
|
|
3728
4114
|
case 'outdated': await cmdOutdated(); break;
|
|
3729
4115
|
case 'sync-installed': case 'sync': await cmdSyncInstalled(); break;
|
|
3730
4116
|
case 'tags': await cmdTags(); break;
|