primitive-admin 1.0.4 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/primitive.js +4 -0
- package/dist/bin/primitive.js.map +1 -1
- package/dist/src/commands/comparisons.js +108 -0
- package/dist/src/commands/comparisons.js.map +1 -0
- package/dist/src/commands/llm.js +241 -0
- package/dist/src/commands/llm.js.map +1 -0
- package/dist/src/commands/prompts.js +170 -1
- package/dist/src/commands/prompts.js.map +1 -1
- package/dist/src/commands/workflows.js +169 -1
- package/dist/src/commands/workflows.js.map +1 -1
- package/dist/src/lib/api-client.js +33 -0
- package/dist/src/lib/api-client.js.map +1 -1
- package/package.json +5 -2
package/dist/bin/primitive.js
CHANGED
|
@@ -11,6 +11,8 @@ import { registerAdminsCommands } from "../src/commands/admins.js";
|
|
|
11
11
|
import { registerCatalogCommands } from "../src/commands/catalog.js";
|
|
12
12
|
import { registerAnalyticsCommands } from "../src/commands/analytics.js";
|
|
13
13
|
import { registerSyncCommands } from "../src/commands/sync.js";
|
|
14
|
+
import { registerLlmCommands } from "../src/commands/llm.js";
|
|
15
|
+
import { registerComparisonsCommands } from "../src/commands/comparisons.js";
|
|
14
16
|
import { error } from "../src/lib/output.js";
|
|
15
17
|
import { ApiError } from "../src/lib/api-client.js";
|
|
16
18
|
const program = new Command();
|
|
@@ -51,6 +53,8 @@ registerAdminsCommands(program);
|
|
|
51
53
|
registerCatalogCommands(program);
|
|
52
54
|
registerAnalyticsCommands(program);
|
|
53
55
|
registerSyncCommands(program);
|
|
56
|
+
registerLlmCommands(program);
|
|
57
|
+
registerComparisonsCommands(program);
|
|
54
58
|
// Global error handler
|
|
55
59
|
program.hook("preAction", () => {
|
|
56
60
|
// Reset API client state before each command
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"primitive.js","sourceRoot":"","sources":["../../bin/primitive.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AACvE,OAAO,EAAE,4BAA4B,EAAE,MAAM,iCAAiC,CAAC;AAC/E,OAAO,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;AACrE,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;AACrE,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAEpD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC;;;4EAG6D,CAAC;KAC1E,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,OAAO,EAAE;;;;;;;;;;;;;;;;;CAiBvB,CAAC,CAAC;AAEH,8BAA8B;AAC9B,oBAAoB,CAAC,OAAO,CAAC,CAAC;AAC9B,oBAAoB,CAAC,OAAO,CAAC,CAAC;AAC9B,qBAAqB,CAAC,OAAO,CAAC,CAAC;AAC/B,wBAAwB,CAAC,OAAO,CAAC,CAAC;AAClC,4BAA4B,CAAC,OAAO,CAAC,CAAC;AACtC,uBAAuB,CAAC,OAAO,CAAC,CAAC;AACjC,yBAAyB,CAAC,OAAO,CAAC,CAAC;AACnC,sBAAsB,CAAC,OAAO,CAAC,CAAC;AAChC,uBAAuB,CAAC,OAAO,CAAC,CAAC;AACjC,yBAAyB,CAAC,OAAO,CAAC,CAAC;AACnC,oBAAoB,CAAC,OAAO,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"primitive.js","sourceRoot":"","sources":["../../bin/primitive.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AACvE,OAAO,EAAE,4BAA4B,EAAE,MAAM,iCAAiC,CAAC;AAC/E,OAAO,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;AACrE,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;AACrE,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,2BAA2B,EAAE,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAEpD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC;;;4EAG6D,CAAC;KAC1E,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,OAAO,EAAE;;;;;;;;;;;;;;;;;CAiBvB,CAAC,CAAC;AAEH,8BAA8B;AAC9B,oBAAoB,CAAC,OAAO,CAAC,CAAC;AAC9B,oBAAoB,CAAC,OAAO,CAAC,CAAC;AAC9B,qBAAqB,CAAC,OAAO,CAAC,CAAC;AAC/B,wBAAwB,CAAC,OAAO,CAAC,CAAC;AAClC,4BAA4B,CAAC,OAAO,CAAC,CAAC;AACtC,uBAAuB,CAAC,OAAO,CAAC,CAAC;AACjC,yBAAyB,CAAC,OAAO,CAAC,CAAC;AACnC,sBAAsB,CAAC,OAAO,CAAC,CAAC;AAChC,uBAAuB,CAAC,OAAO,CAAC,CAAC;AACjC,yBAAyB,CAAC,OAAO,CAAC,CAAC;AACnC,oBAAoB,CAAC,OAAO,CAAC,CAAC;AAC9B,mBAAmB,CAAC,OAAO,CAAC,CAAC;AAC7B,2BAA2B,CAAC,OAAO,CAAC,CAAC;AAErC,uBAAuB;AACvB,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE;IAC7B,6CAA6C;AAC/C,CAAC,CAAC,CAAC;AAEH,oBAAoB;AACpB,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IAC7C,IAAI,GAAG,YAAY,QAAQ,EAAE,CAAC;QAC5B,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACnB,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;YAC3B,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,GAAG,CAAC,OAAO,IAAI,8BAA8B,CAAC,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import chalk from "chalk";
|
|
2
|
+
import { ApiClient } from "../lib/api-client.js";
|
|
3
|
+
import { getCurrentAppId } from "../lib/config.js";
|
|
4
|
+
import { error, info, keyValue, formatTable, formatId, formatDate, formatDuration, json, divider, } from "../lib/output.js";
|
|
5
|
+
function resolveAppId(appId, options) {
|
|
6
|
+
const resolved = appId || options.app || getCurrentAppId();
|
|
7
|
+
if (!resolved) {
|
|
8
|
+
error("No app specified. Use --app or 'primitive use <app-id>' to set context.");
|
|
9
|
+
process.exit(1);
|
|
10
|
+
}
|
|
11
|
+
return resolved;
|
|
12
|
+
}
|
|
13
|
+
export function registerComparisonsCommands(program) {
|
|
14
|
+
const comparisons = program
|
|
15
|
+
.command("comparisons")
|
|
16
|
+
.description("View and compare test run results grouped by batch")
|
|
17
|
+
.addHelpText("after", `
|
|
18
|
+
Examples:
|
|
19
|
+
$ primitive comparisons get <comparison-group>
|
|
20
|
+
$ primitive comparisons get batch_01ABC123 --json
|
|
21
|
+
`);
|
|
22
|
+
// Get comparison group
|
|
23
|
+
comparisons
|
|
24
|
+
.command("get")
|
|
25
|
+
.description("Get all test runs in a comparison group")
|
|
26
|
+
.argument("<group>", "Comparison group ID (batch ID)")
|
|
27
|
+
.option("--app <app-id>", "App ID (uses current app if not specified)")
|
|
28
|
+
.option("--json", "Output as JSON")
|
|
29
|
+
.action(async (group, options) => {
|
|
30
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
31
|
+
const client = new ApiClient();
|
|
32
|
+
try {
|
|
33
|
+
const result = await client.getComparisonGroup(resolvedAppId, group);
|
|
34
|
+
if (options.json) {
|
|
35
|
+
json(result);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
keyValue("Comparison Group", result.comparisonGroup);
|
|
39
|
+
keyValue("Total Runs", result.runs?.length || 0);
|
|
40
|
+
if (!result.runs || result.runs.length === 0) {
|
|
41
|
+
info("No runs found in this comparison group.");
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
// Calculate summary
|
|
45
|
+
const passed = result.runs.filter((r) => r.verificationPassed === true).length;
|
|
46
|
+
const failed = result.runs.filter((r) => r.verificationPassed === false).length;
|
|
47
|
+
const pending = result.runs.filter((r) => r.verificationPassed === null || r.verificationPassed === undefined).length;
|
|
48
|
+
keyValue("Passed", chalk.green(passed));
|
|
49
|
+
keyValue("Failed", failed > 0 ? chalk.red(failed) : "0");
|
|
50
|
+
if (pending > 0) {
|
|
51
|
+
keyValue("Pending", chalk.dim(pending));
|
|
52
|
+
}
|
|
53
|
+
divider();
|
|
54
|
+
info("Runs:");
|
|
55
|
+
console.log(formatTable(result.runs, [
|
|
56
|
+
{ header: "RUN ID", key: "runId", format: formatId },
|
|
57
|
+
{ header: "TEST CASE", key: "testCaseId", format: (v) => v ? formatId(v) : "-" },
|
|
58
|
+
{ header: "BLOCK", key: "blockType" },
|
|
59
|
+
{
|
|
60
|
+
header: "PASSED",
|
|
61
|
+
key: "verificationPassed",
|
|
62
|
+
format: (v) => v === true
|
|
63
|
+
? chalk.green("Yes")
|
|
64
|
+
: v === false
|
|
65
|
+
? chalk.red("No")
|
|
66
|
+
: chalk.dim("-"),
|
|
67
|
+
},
|
|
68
|
+
{ header: "DURATION", key: "durationMs", format: formatDuration },
|
|
69
|
+
{ header: "CONFIG", key: "configId", format: (v) => v ? formatId(v) : "-" },
|
|
70
|
+
{ header: "STARTED", key: "startedAt", format: formatDate },
|
|
71
|
+
]));
|
|
72
|
+
// Show detailed failures if any
|
|
73
|
+
const failures = result.runs.filter((r) => r.verificationPassed === false);
|
|
74
|
+
if (failures.length > 0) {
|
|
75
|
+
divider();
|
|
76
|
+
info("Failed Runs Details:");
|
|
77
|
+
for (const run of failures) {
|
|
78
|
+
console.log(`\n ${chalk.red("✗")} Run ${formatId(run.runId)}`);
|
|
79
|
+
if (run.verificationDetails) {
|
|
80
|
+
const details = typeof run.verificationDetails === "string"
|
|
81
|
+
? JSON.parse(run.verificationDetails)
|
|
82
|
+
: run.verificationDetails;
|
|
83
|
+
if (details.reason) {
|
|
84
|
+
console.log(` Reason: ${details.reason}`);
|
|
85
|
+
}
|
|
86
|
+
if (details.checks) {
|
|
87
|
+
for (const check of details.checks) {
|
|
88
|
+
if (!check.passed) {
|
|
89
|
+
const icon = chalk.red("✗");
|
|
90
|
+
console.log(` ${icon} ${check.name}: ${check.message || "Failed"}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
// Exit with error if any tests failed
|
|
98
|
+
if (failed > 0) {
|
|
99
|
+
process.exit(1);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
catch (err) {
|
|
103
|
+
error(err.message);
|
|
104
|
+
process.exit(1);
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=comparisons.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"comparisons.js","sourceRoot":"","sources":["../../../src/commands/comparisons.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAEL,KAAK,EACL,IAAI,EACJ,QAAQ,EACR,WAAW,EACX,QAAQ,EACR,UAAU,EAEV,cAAc,EACd,IAAI,EACJ,OAAO,GACR,MAAM,kBAAkB,CAAC;AAE1B,SAAS,YAAY,CAAC,KAAyB,EAAE,OAAY;IAC3D,MAAM,QAAQ,GAAG,KAAK,IAAI,OAAO,CAAC,GAAG,IAAI,eAAe,EAAE,CAAC;IAC3D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,KAAK,CAAC,yEAAyE,CAAC,CAAC;QACjF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,OAAgB;IAC1D,MAAM,WAAW,GAAG,OAAO;SACxB,OAAO,CAAC,aAAa,CAAC;SACtB,WAAW,CAAC,oDAAoD,CAAC;SACjE,WAAW,CAAC,OAAO,EAAE;;;;CAIzB,CAAC,CAAC;IAED,uBAAuB;IACvB,WAAW;SACR,OAAO,CAAC,KAAK,CAAC;SACd,WAAW,CAAC,yCAAyC,CAAC;SACtD,QAAQ,CAAC,SAAS,EAAE,gCAAgC,CAAC;SACrD,MAAM,CAAC,gBAAgB,EAAE,4CAA4C,CAAC;SACtE,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC;SAClC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAC/B,MAAM,aAAa,GAAG,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAE/B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;YAErE,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,IAAI,CAAC,MAAM,CAAC,CAAC;gBACb,OAAO;YACT,CAAC;YAED,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;YACrD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAEjD,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC7C,IAAI,CAAC,yCAAyC,CAAC,CAAC;gBAChD,OAAO;YACT,CAAC;YAED,oBAAoB;YACpB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;YACpF,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;YACrF,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,KAAK,IAAI,IAAI,CAAC,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;YAE3H,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YACxC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACzD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;YAC1C,CAAC;YAED,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,OAAO,CAAC,CAAC;YAEd,OAAO,CAAC,GAAG,CACT,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE;gBACvB,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE;gBACpD,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE;gBACxF,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE;gBACrC;oBACE,MAAM,EAAE,QAAQ;oBAChB,GAAG,EAAE,oBAAoB;oBACzB,MAAM,EAAE,CAAC,CAAiB,EAAE,EAAE,CAC5B,CAAC,KAAK,IAAI;wBACR,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;wBACpB,CAAC,CAAC,CAAC,KAAK,KAAK;4BACb,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;4BACjB,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;iBACrB;gBACD,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,cAAc,EAAE;gBACjE,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE;gBACnF,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE;aAC5D,CAAC,CACH,CAAC;YAEF,gCAAgC;YAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,KAAK,KAAK,CAAC,CAAC;YAChF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,OAAO,EAAE,CAAC;gBACV,IAAI,CAAC,sBAAsB,CAAC,CAAC;gBAC7B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC3B,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAChE,IAAI,GAAG,CAAC,mBAAmB,EAAE,CAAC;wBAC5B,MAAM,OAAO,GAAG,OAAO,GAAG,CAAC,mBAAmB,KAAK,QAAQ;4BACzD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC;4BACrC,CAAC,CAAC,GAAG,CAAC,mBAAmB,CAAC;wBAC5B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;4BACnB,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;wBAC/C,CAAC;wBACD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;4BACnB,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gCACnC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;oCAClB,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oCAC5B,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC;gCACzE,CAAC;4BACH,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,sCAAsC;YACtC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
import { ApiClient } from "../lib/api-client.js";
|
|
2
|
+
import { getCurrentAppId } from "../lib/config.js";
|
|
3
|
+
import { success, error, info, keyValue, formatTable, json, divider, } from "../lib/output.js";
|
|
4
|
+
function resolveAppId(appId, options) {
|
|
5
|
+
const resolved = appId || options.app || getCurrentAppId();
|
|
6
|
+
if (!resolved) {
|
|
7
|
+
error("No app specified. Use --app or 'primitive use <app-id>' to set context.");
|
|
8
|
+
process.exit(1);
|
|
9
|
+
}
|
|
10
|
+
return resolved;
|
|
11
|
+
}
|
|
12
|
+
export function registerLlmCommands(program) {
|
|
13
|
+
const llm = program
|
|
14
|
+
.command("llm")
|
|
15
|
+
.description("AI-powered generation tools for prompts and evaluators")
|
|
16
|
+
.addHelpText("after", `
|
|
17
|
+
Examples:
|
|
18
|
+
$ primitive llm models --provider openrouter
|
|
19
|
+
$ primitive llm generate-prompt --description "Summarize documents"
|
|
20
|
+
$ primitive llm generate-evaluator --prompt-id 01HXY...
|
|
21
|
+
$ primitive llm generate-evaluator-workflow --workflow-id 01ABC...
|
|
22
|
+
`);
|
|
23
|
+
// List available models
|
|
24
|
+
llm
|
|
25
|
+
.command("models")
|
|
26
|
+
.description("List available LLM models")
|
|
27
|
+
.option("--provider <provider>", "Provider: openrouter, gemini", "openrouter")
|
|
28
|
+
.option("--json", "Output as JSON")
|
|
29
|
+
.action(async (options) => {
|
|
30
|
+
const client = new ApiClient();
|
|
31
|
+
try {
|
|
32
|
+
const result = await client.listLlmModels(options.provider);
|
|
33
|
+
if (options.json) {
|
|
34
|
+
json(result);
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
info(`Available models for ${options.provider}:`);
|
|
38
|
+
keyValue("Default Model", result.defaultModel || "-");
|
|
39
|
+
keyValue("Total Models", result.models?.length || 0);
|
|
40
|
+
if (result.models && result.models.length > 0) {
|
|
41
|
+
divider();
|
|
42
|
+
// Show first 30 models
|
|
43
|
+
const displayModels = result.models.slice(0, 30);
|
|
44
|
+
console.log(formatTable(displayModels, [
|
|
45
|
+
{ header: "MODEL ID", key: "id" },
|
|
46
|
+
{ header: "NAME", key: "name" },
|
|
47
|
+
{
|
|
48
|
+
header: "CONTEXT",
|
|
49
|
+
key: "context_length",
|
|
50
|
+
format: (v) => v ? `${Math.round(v / 1000)}K` : "-",
|
|
51
|
+
},
|
|
52
|
+
]));
|
|
53
|
+
if (result.models.length > 30) {
|
|
54
|
+
info(`... and ${result.models.length - 30} more. Use --json for full list.`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
catch (err) {
|
|
59
|
+
error(err.message);
|
|
60
|
+
process.exit(1);
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
// Generate prompt from description
|
|
64
|
+
llm
|
|
65
|
+
.command("generate-prompt")
|
|
66
|
+
.description("Generate a prompt configuration from a natural language description")
|
|
67
|
+
.option("--description <text>", "Description of what the prompt should do")
|
|
68
|
+
.option("--with-output-schema", "Also generate an output schema")
|
|
69
|
+
.option("--json", "Output as JSON")
|
|
70
|
+
.action(async (options) => {
|
|
71
|
+
let description = options.description;
|
|
72
|
+
// If no description provided, prompt interactively
|
|
73
|
+
if (!description) {
|
|
74
|
+
const inquirer = await import("inquirer");
|
|
75
|
+
const { desc } = await inquirer.default.prompt([
|
|
76
|
+
{
|
|
77
|
+
type: "input",
|
|
78
|
+
name: "desc",
|
|
79
|
+
message: "Describe what the prompt should do:",
|
|
80
|
+
validate: (input) => input.trim().length > 0 || "Description is required",
|
|
81
|
+
},
|
|
82
|
+
]);
|
|
83
|
+
description = desc;
|
|
84
|
+
}
|
|
85
|
+
const client = new ApiClient();
|
|
86
|
+
try {
|
|
87
|
+
if (!options.json) {
|
|
88
|
+
info("Generating prompt configuration...");
|
|
89
|
+
}
|
|
90
|
+
const result = await client.generatePrompt({
|
|
91
|
+
description,
|
|
92
|
+
generateOutputSchema: options.withOutputSchema || false,
|
|
93
|
+
});
|
|
94
|
+
if (options.json) {
|
|
95
|
+
json(result);
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
if (result.error) {
|
|
99
|
+
error(`Generation failed: ${result.error}`);
|
|
100
|
+
process.exit(1);
|
|
101
|
+
}
|
|
102
|
+
success("Prompt configuration generated!");
|
|
103
|
+
divider();
|
|
104
|
+
keyValue("Key", result.promptKey);
|
|
105
|
+
keyValue("Name", result.displayName);
|
|
106
|
+
keyValue("Description", result.description);
|
|
107
|
+
divider();
|
|
108
|
+
info("System Prompt:");
|
|
109
|
+
console.log(result.systemPrompt || "(none)");
|
|
110
|
+
divider();
|
|
111
|
+
info("User Prompt Template:");
|
|
112
|
+
console.log(result.userPromptTemplate);
|
|
113
|
+
divider();
|
|
114
|
+
keyValue("Temperature", result.temperature);
|
|
115
|
+
keyValue("Top P", result.topP || "-");
|
|
116
|
+
keyValue("Max Tokens", result.maxTokens || "-");
|
|
117
|
+
if (result.inputSchema) {
|
|
118
|
+
divider();
|
|
119
|
+
info("Input Schema:");
|
|
120
|
+
console.log(JSON.stringify(result.inputSchema, null, 2));
|
|
121
|
+
}
|
|
122
|
+
if (result.outputSchema) {
|
|
123
|
+
divider();
|
|
124
|
+
info("Output Schema:");
|
|
125
|
+
console.log(JSON.stringify(result.outputSchema, null, 2));
|
|
126
|
+
}
|
|
127
|
+
divider();
|
|
128
|
+
info("To create this prompt, copy the JSON output (--json) and use:");
|
|
129
|
+
console.log(" primitive prompts create --from-file <file.toml>");
|
|
130
|
+
}
|
|
131
|
+
catch (err) {
|
|
132
|
+
error(err.message);
|
|
133
|
+
process.exit(1);
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
// Generate evaluator for a prompt
|
|
137
|
+
llm
|
|
138
|
+
.command("generate-evaluator")
|
|
139
|
+
.description("Generate an evaluator prompt for an existing prompt")
|
|
140
|
+
.option("--app <app-id>", "App ID (uses current app if not specified)")
|
|
141
|
+
.requiredOption("--prompt-id <prompt-id>", "Source prompt ID to create evaluator for")
|
|
142
|
+
.option("--json", "Output as JSON")
|
|
143
|
+
.action(async (options) => {
|
|
144
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
145
|
+
const client = new ApiClient();
|
|
146
|
+
try {
|
|
147
|
+
if (!options.json) {
|
|
148
|
+
info("Generating evaluator prompt...");
|
|
149
|
+
}
|
|
150
|
+
const result = await client.generateEvaluator({
|
|
151
|
+
appId: resolvedAppId,
|
|
152
|
+
promptId: options.promptId,
|
|
153
|
+
});
|
|
154
|
+
if (options.json) {
|
|
155
|
+
json(result);
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
if (result.error) {
|
|
159
|
+
error(`Generation failed: ${result.error}`);
|
|
160
|
+
process.exit(1);
|
|
161
|
+
}
|
|
162
|
+
success("Evaluator prompt configuration generated!");
|
|
163
|
+
divider();
|
|
164
|
+
keyValue("Key", result.promptKey);
|
|
165
|
+
keyValue("Name", result.displayName);
|
|
166
|
+
keyValue("Description", result.description);
|
|
167
|
+
divider();
|
|
168
|
+
info("System Prompt:");
|
|
169
|
+
console.log(result.systemPrompt || "(none)");
|
|
170
|
+
divider();
|
|
171
|
+
info("User Prompt Template:");
|
|
172
|
+
console.log(result.userPromptTemplate);
|
|
173
|
+
divider();
|
|
174
|
+
if (result.outputSchema) {
|
|
175
|
+
info("Output Schema:");
|
|
176
|
+
console.log(JSON.stringify(result.outputSchema, null, 2));
|
|
177
|
+
}
|
|
178
|
+
divider();
|
|
179
|
+
info("To create this evaluator, copy the JSON output (--json) and use:");
|
|
180
|
+
console.log(" primitive prompts create --from-file <file.toml>");
|
|
181
|
+
info("Then link it to test cases using --evaluator-prompt option.");
|
|
182
|
+
}
|
|
183
|
+
catch (err) {
|
|
184
|
+
error(err.message);
|
|
185
|
+
process.exit(1);
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
// Generate evaluator for a workflow
|
|
189
|
+
llm
|
|
190
|
+
.command("generate-evaluator-workflow")
|
|
191
|
+
.description("Generate an evaluator prompt for an existing workflow")
|
|
192
|
+
.option("--app <app-id>", "App ID (uses current app if not specified)")
|
|
193
|
+
.requiredOption("--workflow-id <workflow-id>", "Source workflow ID to create evaluator for")
|
|
194
|
+
.option("--json", "Output as JSON")
|
|
195
|
+
.action(async (options) => {
|
|
196
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
197
|
+
const client = new ApiClient();
|
|
198
|
+
try {
|
|
199
|
+
if (!options.json) {
|
|
200
|
+
info("Generating workflow evaluator prompt...");
|
|
201
|
+
}
|
|
202
|
+
const result = await client.generateWorkflowEvaluator({
|
|
203
|
+
appId: resolvedAppId,
|
|
204
|
+
workflowId: options.workflowId,
|
|
205
|
+
});
|
|
206
|
+
if (options.json) {
|
|
207
|
+
json(result);
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
if (result.error) {
|
|
211
|
+
error(`Generation failed: ${result.error}`);
|
|
212
|
+
process.exit(1);
|
|
213
|
+
}
|
|
214
|
+
success("Workflow evaluator prompt configuration generated!");
|
|
215
|
+
divider();
|
|
216
|
+
keyValue("Key", result.promptKey);
|
|
217
|
+
keyValue("Name", result.displayName);
|
|
218
|
+
keyValue("Description", result.description);
|
|
219
|
+
divider();
|
|
220
|
+
info("System Prompt:");
|
|
221
|
+
console.log(result.systemPrompt || "(none)");
|
|
222
|
+
divider();
|
|
223
|
+
info("User Prompt Template:");
|
|
224
|
+
console.log(result.userPromptTemplate);
|
|
225
|
+
divider();
|
|
226
|
+
if (result.outputSchema) {
|
|
227
|
+
info("Output Schema:");
|
|
228
|
+
console.log(JSON.stringify(result.outputSchema, null, 2));
|
|
229
|
+
}
|
|
230
|
+
divider();
|
|
231
|
+
info("To create this evaluator, copy the JSON output (--json) and use:");
|
|
232
|
+
console.log(" primitive prompts create --from-file <file.toml>");
|
|
233
|
+
info("Then link it to workflow test cases using --evaluator-prompt option.");
|
|
234
|
+
}
|
|
235
|
+
catch (err) {
|
|
236
|
+
error(err.message);
|
|
237
|
+
process.exit(1);
|
|
238
|
+
}
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
//# sourceMappingURL=llm.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../../src/commands/llm.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EACL,OAAO,EACP,KAAK,EACL,IAAI,EAEJ,QAAQ,EACR,WAAW,EAEX,IAAI,EACJ,OAAO,GACR,MAAM,kBAAkB,CAAC;AAE1B,SAAS,YAAY,CAAC,KAAyB,EAAE,OAAY;IAC3D,MAAM,QAAQ,GAAG,KAAK,IAAI,OAAO,CAAC,GAAG,IAAI,eAAe,EAAE,CAAC;IAC3D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,KAAK,CAAC,yEAAyE,CAAC,CAAC;QACjF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,OAAgB;IAClD,MAAM,GAAG,GAAG,OAAO;SAChB,OAAO,CAAC,KAAK,CAAC;SACd,WAAW,CAAC,wDAAwD,CAAC;SACrE,WAAW,CAAC,OAAO,EAAE;;;;;;CAMzB,CAAC,CAAC;IAED,wBAAwB;IACxB,GAAG;SACA,OAAO,CAAC,QAAQ,CAAC;SACjB,WAAW,CAAC,2BAA2B,CAAC;SACxC,MAAM,CAAC,uBAAuB,EAAE,8BAA8B,EAAE,YAAY,CAAC;SAC7E,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC;SAClC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QACxB,MAAM,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAE/B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAE5D,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,IAAI,CAAC,MAAM,CAAC,CAAC;gBACb,OAAO;YACT,CAAC;YAED,IAAI,CAAC,wBAAwB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YAClD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC,YAAY,IAAI,GAAG,CAAC,CAAC;YACtD,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAErD,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9C,OAAO,EAAE,CAAC;gBACV,uBAAuB;gBACvB,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACjD,OAAO,CAAC,GAAG,CACT,WAAW,CAAC,aAAa,EAAE;oBACzB,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,EAAE;oBACjC,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE;oBAC/B;wBACE,MAAM,EAAE,SAAS;wBACjB,GAAG,EAAE,gBAAgB;wBACrB,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;qBAC5D;iBACF,CAAC,CACH,CAAC;gBACF,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;oBAC9B,IAAI,CAAC,WAAW,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,EAAE,kCAAkC,CAAC,CAAC;gBAC/E,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,mCAAmC;IACnC,GAAG;SACA,OAAO,CAAC,iBAAiB,CAAC;SAC1B,WAAW,CAAC,qEAAqE,CAAC;SAClF,MAAM,CAAC,sBAAsB,EAAE,0CAA0C,CAAC;SAC1E,MAAM,CAAC,sBAAsB,EAAE,gCAAgC,CAAC;SAChE,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC;SAClC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QACxB,IAAI,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QAEtC,mDAAmD;QACnD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;YAC1C,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC;gBAC7C;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,qCAAqC;oBAC9C,QAAQ,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,yBAAyB;iBAClF;aACF,CAAC,CAAC;YACH,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAE/B,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAClB,IAAI,CAAC,oCAAoC,CAAC,CAAC;YAC7C,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC;gBACzC,WAAW;gBACX,oBAAoB,EAAE,OAAO,CAAC,gBAAgB,IAAI,KAAK;aACxD,CAAC,CAAC;YAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,IAAI,CAAC,MAAM,CAAC,CAAC;gBACb,OAAO;YACT,CAAC;YAED,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,KAAK,CAAC,sBAAsB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YAED,OAAO,CAAC,iCAAiC,CAAC,CAAC;YAC3C,OAAO,EAAE,CAAC;YACV,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YACrC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YAC5C,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,QAAQ,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;YACvC,OAAO,EAAE,CAAC;YACV,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YAC5C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;YACtC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC;YAEhD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;gBACvB,OAAO,EAAE,CAAC;gBACV,IAAI,CAAC,eAAe,CAAC,CAAC;gBACtB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3D,CAAC;YAED,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;gBACxB,OAAO,EAAE,CAAC;gBACV,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5D,CAAC;YAED,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,+DAA+D,CAAC,CAAC;YACtE,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QACpE,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,kCAAkC;IAClC,GAAG;SACA,OAAO,CAAC,oBAAoB,CAAC;SAC7B,WAAW,CAAC,qDAAqD,CAAC;SAClE,MAAM,CAAC,gBAAgB,EAAE,4CAA4C,CAAC;SACtE,cAAc,CAAC,yBAAyB,EAAE,0CAA0C,CAAC;SACrF,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC;SAClC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QACxB,MAAM,aAAa,GAAG,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAE/B,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAClB,IAAI,CAAC,gCAAgC,CAAC,CAAC;YACzC,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC;gBAC5C,KAAK,EAAE,aAAa;gBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;aAC3B,CAAC,CAAC;YAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,IAAI,CAAC,MAAM,CAAC,CAAC;gBACb,OAAO;YACT,CAAC;YAED,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,KAAK,CAAC,sBAAsB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YAED,OAAO,CAAC,2CAA2C,CAAC,CAAC;YACrD,OAAO,EAAE,CAAC;YACV,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YACrC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YAC5C,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,QAAQ,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;YACvC,OAAO,EAAE,CAAC;YAEV,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;gBACxB,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5D,CAAC;YAED,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,kEAAkE,CAAC,CAAC;YACzE,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;YAClE,IAAI,CAAC,6DAA6D,CAAC,CAAC;QACtE,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,oCAAoC;IACpC,GAAG;SACA,OAAO,CAAC,6BAA6B,CAAC;SACtC,WAAW,CAAC,uDAAuD,CAAC;SACpE,MAAM,CAAC,gBAAgB,EAAE,4CAA4C,CAAC;SACtE,cAAc,CAAC,6BAA6B,EAAE,4CAA4C,CAAC;SAC3F,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC;SAClC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QACxB,MAAM,aAAa,GAAG,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAE/B,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAClB,IAAI,CAAC,yCAAyC,CAAC,CAAC;YAClD,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC;gBACpD,KAAK,EAAE,aAAa;gBACpB,UAAU,EAAE,OAAO,CAAC,UAAU;aAC/B,CAAC,CAAC;YAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,IAAI,CAAC,MAAM,CAAC,CAAC;gBACb,OAAO;YACT,CAAC;YAED,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,KAAK,CAAC,sBAAsB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YAED,OAAO,CAAC,oDAAoD,CAAC,CAAC;YAC9D,OAAO,EAAE,CAAC;YACV,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YACrC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YAC5C,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,QAAQ,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;YACvC,OAAO,EAAE,CAAC;YAEV,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;gBACxB,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5D,CAAC;YAED,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,kEAAkE,CAAC,CAAC;YACzE,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;YAClE,IAAI,CAAC,sEAAsE,CAAC,CAAC;QAC/E,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -5,7 +5,7 @@ import { lookup as mimeLookup } from "mime-types";
|
|
|
5
5
|
import { ApiClient } from "../lib/api-client.js";
|
|
6
6
|
import { getCurrentAppId } from "../lib/config.js";
|
|
7
7
|
import chalk from "chalk";
|
|
8
|
-
import { success, error, info, keyValue, formatTable, formatId, formatDate, formatStatus, formatDuration, json, divider, } from "../lib/output.js";
|
|
8
|
+
import { success, error, info, warn, keyValue, formatTable, formatId, formatDate, formatStatus, formatDuration, json, divider, } from "../lib/output.js";
|
|
9
9
|
function resolveAppId(appId, options) {
|
|
10
10
|
const resolved = appId || options.app || getCurrentAppId();
|
|
11
11
|
if (!resolved) {
|
|
@@ -1079,6 +1079,175 @@ Examples:
|
|
|
1079
1079
|
}
|
|
1080
1080
|
});
|
|
1081
1081
|
// ============================================
|
|
1082
|
+
// BATCH TEST EXECUTION SUBCOMMAND
|
|
1083
|
+
// ============================================
|
|
1084
|
+
const batch = tests
|
|
1085
|
+
.command("batch")
|
|
1086
|
+
.description("Run tests in parallel using workflow-based execution")
|
|
1087
|
+
.addHelpText("after", `
|
|
1088
|
+
Examples:
|
|
1089
|
+
$ primitive prompts tests batch start <prompt-id>
|
|
1090
|
+
$ primitive prompts tests batch start <prompt-id> --test-cases 01ABC,01DEF
|
|
1091
|
+
$ primitive prompts tests batch status <prompt-id> <batch-id>
|
|
1092
|
+
$ primitive prompts tests batch cancel <prompt-id> <batch-id>
|
|
1093
|
+
`);
|
|
1094
|
+
// Start batch tests
|
|
1095
|
+
batch
|
|
1096
|
+
.command("start")
|
|
1097
|
+
.description("Start parallel batch test execution")
|
|
1098
|
+
.argument("<prompt-id>", "Prompt ID")
|
|
1099
|
+
.option("--app <app-id>", "App ID (uses current app if not specified)")
|
|
1100
|
+
.option("--config <config-id>", "Override config ID for all tests")
|
|
1101
|
+
.option("--test-cases <ids>", "Comma-separated list of test case IDs to run")
|
|
1102
|
+
.option("--json", "Output as JSON")
|
|
1103
|
+
.action(async (promptId, options) => {
|
|
1104
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
1105
|
+
const client = new ApiClient();
|
|
1106
|
+
try {
|
|
1107
|
+
let testCaseIds;
|
|
1108
|
+
if (options.testCases) {
|
|
1109
|
+
testCaseIds = options.testCases.split(",").map((id) => id.trim()).filter(Boolean);
|
|
1110
|
+
}
|
|
1111
|
+
if (!options.json) {
|
|
1112
|
+
info("Starting batch test execution...");
|
|
1113
|
+
}
|
|
1114
|
+
const result = await client.startBatchTests(resolvedAppId, "prompt", promptId, {
|
|
1115
|
+
overrideConfigId: options.config,
|
|
1116
|
+
testCaseIds,
|
|
1117
|
+
});
|
|
1118
|
+
if (options.json) {
|
|
1119
|
+
json(result);
|
|
1120
|
+
return;
|
|
1121
|
+
}
|
|
1122
|
+
success("Batch tests started!");
|
|
1123
|
+
keyValue("Batch ID", result.batchId);
|
|
1124
|
+
keyValue("Total Tests", result.totalTests);
|
|
1125
|
+
keyValue("Workflow Instances", result.instanceIds?.length || 0);
|
|
1126
|
+
if (result.errors && result.errors.length > 0) {
|
|
1127
|
+
divider();
|
|
1128
|
+
warn(`${result.errors.length} test(s) failed to start`);
|
|
1129
|
+
}
|
|
1130
|
+
divider();
|
|
1131
|
+
info("Check status with:");
|
|
1132
|
+
console.log(` primitive prompts tests batch status ${promptId} ${result.batchId}`);
|
|
1133
|
+
}
|
|
1134
|
+
catch (err) {
|
|
1135
|
+
error(err.message);
|
|
1136
|
+
process.exit(1);
|
|
1137
|
+
}
|
|
1138
|
+
});
|
|
1139
|
+
// Get batch status
|
|
1140
|
+
batch
|
|
1141
|
+
.command("status")
|
|
1142
|
+
.description("Get status of a batch test execution")
|
|
1143
|
+
.argument("<prompt-id>", "Prompt ID")
|
|
1144
|
+
.argument("<batch-id>", "Batch ID")
|
|
1145
|
+
.option("--app <app-id>", "App ID (uses current app if not specified)")
|
|
1146
|
+
.option("--wait", "Wait for completion (polls every 2 seconds)")
|
|
1147
|
+
.option("--json", "Output as JSON")
|
|
1148
|
+
.action(async (promptId, batchId, options) => {
|
|
1149
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
1150
|
+
const client = new ApiClient();
|
|
1151
|
+
try {
|
|
1152
|
+
const fetchStatus = async () => {
|
|
1153
|
+
return client.getBatchTestStatus(resolvedAppId, "prompt", promptId, batchId);
|
|
1154
|
+
};
|
|
1155
|
+
let result = await fetchStatus();
|
|
1156
|
+
if (options.wait && result.status === "running") {
|
|
1157
|
+
info("Waiting for batch completion...");
|
|
1158
|
+
while (result.status === "running") {
|
|
1159
|
+
await new Promise((r) => setTimeout(r, 2000));
|
|
1160
|
+
result = await fetchStatus();
|
|
1161
|
+
process.stdout.write(`\r Completed: ${result.completed}/${result.results?.length || 0} `);
|
|
1162
|
+
}
|
|
1163
|
+
console.log();
|
|
1164
|
+
}
|
|
1165
|
+
if (options.json) {
|
|
1166
|
+
json(result);
|
|
1167
|
+
return;
|
|
1168
|
+
}
|
|
1169
|
+
keyValue("Batch ID", result.batchId);
|
|
1170
|
+
keyValue("Status", formatStatus(result.status));
|
|
1171
|
+
keyValue("Completed", result.completed);
|
|
1172
|
+
keyValue("Passed", chalk.green(result.passed));
|
|
1173
|
+
keyValue("Failed", result.failed > 0 ? chalk.red(result.failed) : "0");
|
|
1174
|
+
if (result.results && result.results.length > 0) {
|
|
1175
|
+
divider();
|
|
1176
|
+
info("Results:");
|
|
1177
|
+
for (const r of result.results) {
|
|
1178
|
+
const passed = r.verificationPassed;
|
|
1179
|
+
const icon = passed ? chalk.green("✓") : passed === false ? chalk.red("✗") : chalk.dim("○");
|
|
1180
|
+
const status = r.status === "completed" ? (passed ? "PASSED" : "FAILED") : r.status;
|
|
1181
|
+
const duration = r.durationMs ? formatDuration(r.durationMs) : "";
|
|
1182
|
+
console.log(` ${icon} ${formatId(r.testCaseId)} - ${status} ${duration}`);
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
// Exit with error if tests failed
|
|
1186
|
+
if (result.failed > 0) {
|
|
1187
|
+
process.exit(1);
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
catch (err) {
|
|
1191
|
+
error(err.message);
|
|
1192
|
+
process.exit(1);
|
|
1193
|
+
}
|
|
1194
|
+
});
|
|
1195
|
+
// Cancel batch tests
|
|
1196
|
+
batch
|
|
1197
|
+
.command("cancel")
|
|
1198
|
+
.description("Cancel a running batch test execution")
|
|
1199
|
+
.argument("<prompt-id>", "Prompt ID")
|
|
1200
|
+
.argument("<batch-id>", "Batch ID")
|
|
1201
|
+
.option("--app <app-id>", "App ID (uses current app if not specified)")
|
|
1202
|
+
.option("-y, --yes", "Skip confirmation prompt")
|
|
1203
|
+
.option("--json", "Output as JSON")
|
|
1204
|
+
.action(async (promptId, batchId, options) => {
|
|
1205
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
1206
|
+
if (!options.yes) {
|
|
1207
|
+
const inquirer = await import("inquirer");
|
|
1208
|
+
const { confirm } = await inquirer.default.prompt([
|
|
1209
|
+
{
|
|
1210
|
+
type: "confirm",
|
|
1211
|
+
name: "confirm",
|
|
1212
|
+
message: `Are you sure you want to cancel batch ${batchId}?`,
|
|
1213
|
+
default: false,
|
|
1214
|
+
},
|
|
1215
|
+
]);
|
|
1216
|
+
if (!confirm) {
|
|
1217
|
+
info("Cancelled.");
|
|
1218
|
+
return;
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
const client = new ApiClient();
|
|
1222
|
+
try {
|
|
1223
|
+
// First get current status to get instance IDs
|
|
1224
|
+
const status = await client.getBatchTestStatus(resolvedAppId, "prompt", promptId, batchId);
|
|
1225
|
+
// Get instance IDs that are still running
|
|
1226
|
+
const runningInstanceIds = status.results
|
|
1227
|
+
?.filter((r) => r.status === "running" || r.status === "pending")
|
|
1228
|
+
.map((r) => r.runId)
|
|
1229
|
+
.filter(Boolean) || [];
|
|
1230
|
+
if (runningInstanceIds.length === 0) {
|
|
1231
|
+
info("No running tests to cancel.");
|
|
1232
|
+
return;
|
|
1233
|
+
}
|
|
1234
|
+
const result = await client.cancelBatchTests(resolvedAppId, "prompt", promptId, batchId, runningInstanceIds);
|
|
1235
|
+
if (options.json) {
|
|
1236
|
+
json(result);
|
|
1237
|
+
return;
|
|
1238
|
+
}
|
|
1239
|
+
success("Batch tests cancelled.");
|
|
1240
|
+
keyValue("Terminated", result.terminated?.length || 0);
|
|
1241
|
+
if (result.errors && result.errors.length > 0) {
|
|
1242
|
+
warn(`${result.errors.length} instance(s) could not be terminated`);
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
catch (err) {
|
|
1246
|
+
error(err.message);
|
|
1247
|
+
process.exit(1);
|
|
1248
|
+
}
|
|
1249
|
+
});
|
|
1250
|
+
// ============================================
|
|
1082
1251
|
// TEST CASE ATTACHMENTS SUBCOMMAND
|
|
1083
1252
|
// ============================================
|
|
1084
1253
|
const attachments = tests
|