vskill 0.4.23 → 0.5.0

package/dist/eval/verdict.d.ts

@@ -1,3 +1,3 @@
-export type EvalVerdict = "EFFECTIVE" | "MARGINAL" | "INEFFECTIVE" | "DEGRADING";
+export type EvalVerdict = "EFFECTIVE" | "MARGINAL" | "INEFFECTIVE" | "EMERGING" | "DEGRADING";
 export declare function computeVerdict(assertionPassRate: number, skillRubricAvg: number, baselineRubricAvg: number): EvalVerdict;
 export declare function verdictColor(verdict: EvalVerdict): string;

package/dist/eval/verdict.js

@@ -1,5 +1,5 @@
 // ---------------------------------------------------------------------------
-// verdict.ts -- four-tier eval verdict computation
+// verdict.ts -- five-tier eval verdict computation
 // ---------------------------------------------------------------------------
 export function computeVerdict(assertionPassRate, skillRubricAvg, baselineRubricAvg) {
     if (assertionPassRate >= 0.8 && skillRubricAvg > baselineRubricAvg + 1) {
@@ -11,6 +11,9 @@ export function computeVerdict(assertionPassRate, skillRubricAvg, baselineRubric
     if (assertionPassRate >= 0.4) {
         return "INEFFECTIVE";
     }
+    if (skillRubricAvg > baselineRubricAvg) {
+        return "EMERGING";
+    }
     return "DEGRADING";
 }
 export function verdictColor(verdict) {
@@ -21,6 +24,8 @@ export function verdictColor(verdict) {
             return "yellow";
         case "INEFFECTIVE":
             return "orange";
+        case "EMERGING":
+            return "cyan";
         case "DEGRADING":
             return "red";
     }

package/dist/eval/verdict.js.map

@@ -1 +1 @@
-{"version":3,"file":"verdict.js","sourceRoot":"","sources":["../../src/eval/verdict.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,mDAAmD;AACnD,8EAA8E;AAI9E,MAAM,UAAU,cAAc,CAC5B,iBAAyB,EACzB,cAAsB,EACtB,iBAAyB;IAEzB,IAAI,iBAAiB,IAAI,GAAG,IAAI,cAAc,GAAG,iBAAiB,GAAG,CAAC,EAAE,CAAC;QACvE,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,IAAI,iBAAiB,IAAI,GAAG,IAAI,cAAc,GAAG,iBAAiB,EAAE,CAAC;QACnE,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,iBAAiB,IAAI,GAAG,EAAE,CAAC;QAC7B,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAoB;IAC/C,QAAQ,OAAO,EAAE,CAAC;QAChB,KAAK,WAAW;YACd,OAAO,OAAO,CAAC;QACjB,KAAK,UAAU;YACb,OAAO,QAAQ,CAAC;QAClB,KAAK,aAAa;YAChB,OAAO,QAAQ,CAAC;QAClB,KAAK,WAAW;YACd,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC"}
+{"version":3,"file":"verdict.js","sourceRoot":"","sources":["../../src/eval/verdict.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,mDAAmD;AACnD,8EAA8E;AAI9E,MAAM,UAAU,cAAc,CAC5B,iBAAyB,EACzB,cAAsB,EACtB,iBAAyB;IAEzB,IAAI,iBAAiB,IAAI,GAAG,IAAI,cAAc,GAAG,iBAAiB,GAAG,CAAC,EAAE,CAAC;QACvE,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,IAAI,iBAAiB,IAAI,GAAG,IAAI,cAAc,GAAG,iBAAiB,EAAE,CAAC;QACnE,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,iBAAiB,IAAI,GAAG,EAAE,CAAC;QAC7B,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,IAAI,cAAc,GAAG,iBAAiB,EAAE,CAAC;QACvC,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAoB;IAC/C,QAAQ,OAAO,EAAE,CAAC;QAChB,KAAK,WAAW;YACd,OAAO,OAAO,CAAC;QACjB,KAAK,UAAU;YACb,OAAO,QAAQ,CAAC;QAClB,KAAK,aAAa;YAChB,OAAO,QAAQ,CAAC;QAClB,KAAK,UAAU;YACb,OAAO,MAAM,CAAC;QAChB,KAAK,WAAW;YACd,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC"}

package/dist/eval-server/error-classifier.d.ts

@@ -1,5 +1,5 @@
 export interface ClassifiedError {
-    category: "rate_limit" | "context_window" | "auth" | "timeout" | "provider_unavailable" | "parse_error" | "unknown";
+    category: "rate_limit" | "context_window" | "auth" | "timeout" | "model_not_found" | "provider_unavailable" | "parse_error" | "unknown";
     title: string;
     description: string;
     hint: string;

package/dist/eval-server/error-classifier.js

@@ -128,6 +128,21 @@ export function classifyError(err, provider) {
             retryable: true,
         };
     }
+    // Model not found — must come before provider_unavailable since both match "not found"
+    if (/model\b.*not found/i.test(msg) && !/command not found/i.test(msg)) {
+        const modelMatch = msg.match(/model\s+"?([^\s"]+)"?/i);
+        const modelName = modelMatch?.[1] || "unknown";
+        const pullHint = provider === "ollama"
+            ? `Pull the model first: ollama pull ${modelName}`
+            : `The model "${modelName}" is not available. Check the model name or try a different one.`;
+        return {
+            category: "model_not_found",
+            title: "Model Not Found",
+            description: `The requested model "${modelName}" is not available.`,
+            hint: pullHint,
+            retryable: false,
+        };
+    }
     // Provider unavailable
     if (matchesAny(msg, PROVIDER_UNAVAILABLE_PATTERNS)) {
         const installHints = {

package/dist/eval-server/error-classifier.js.map

@@ -1 +1 @@
-{"version":3,"file":"error-classifier.js","sourceRoot":"","sources":["../../src/eval-server/error-classifier.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2EAA2E;AAC3E,8EAA8E;AAW9E,8EAA8E;AAC9E,sCAAsC;AACtC,8EAA8E;AAE9E,MAAM,mBAAmB,GAAG;IAC1B,cAAc;IACd,oBAAoB;IACpB,KAAK;IACL,UAAU;IACV,kBAAkB;IAClB,aAAa;IACb,WAAW;CACZ,CAAC;AAEF,MAAM,uBAAuB,GAAG;IAC9B,2BAA2B;IAC3B,8BAA8B;IAC9B,mBAAmB;IACnB,mBAAmB;IACnB,mBAAmB;IACnB,4BAA4B;CAC7B,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,WAAW;IACX,OAAO;IACP,KAAK;IACL,KAAK;IACL,eAAe;IACf,YAAY;IACZ,qBAAqB;IACrB,eAAe;CAChB,CAAC;AAEF,MAAM,gBAAgB,GAAG;IACvB,eAAe;IACf,UAAU;IACV,MAAM;IACN,YAAY;IACZ,QAAQ;IACR,UAAU;CACX,CAAC;AAEF,MAAM,6BAA6B,GAAG;IACpC,QAAQ;IACR,YAAY;IACZ,oBAAoB;IACpB,KAAK;IACL,KAAK;IACL,uBAAuB;IACvB,sBAAsB;IACtB,eAAe;IACf,gBAAgB;CACjB,CAAC;AAEF,MAAM,oBAAoB,GAAG;IAC3B,cAAc;IACd,OAAO;IACP,QAAQ;IACR,iBAAiB;IACjB,mBAAmB;CACpB,CAAC;AAEF,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,SAAS,UAAU,CAAC,GAAW,EAAE,QAAkB;IACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAY,EAAE,QAAiB;IAC3D,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;IAEnE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;QAChB,OAAO;YACL,QAAQ,EAAE,SAAS;YACnB,KAAK,EAAE,eAAe;YACtB,WAAW,EAAE,+CAA+C;YAC5D,IAAI,EAAE,wEAAwE;YAC9E,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,aAAa;IACb,IAAI,UAAU,CAAC,GAAG,EAAE,mBAAmB,CAAC,EAAE,CAAC;QACzC,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,KAAK,EAAE,oBAAoB;YAC3B,WAAW,EAAE,mDAAmD;YAChE,IAAI,EAAE,QAAQ,KAAK,YAAY;gBAC7B,CAAC,CAAC,8DAA8D;gBAChE,CAAC,CAAC,+CAA+C;YACnD,SAAS,EAAE,IAAI;YACf,YAAY,EAAE,MAAM;SACrB,CAAC;IACJ,CAAC;IAED,iBAAiB;IACjB,IAAI,UAAU,CAAC,GAAG,EAAE,uBAAuB,CAAC,EAAE,CAAC;QAC7C,OAAO;YACL,QAAQ,EAAE,gBAAgB;YAC1B,KAAK,EAAE,mBAAmB;YAC1B,WAAW,EAAE,uDAAuD;YACpE,IAAI,EAAE,sFAAsF;YAC5F,SAAS,EAAE,KAAK;SACjB,CAAC;IACJ,CAAC;IAED,OAAO;IACP,IAAI,UAAU,CAAC,GAAG,EAAE,aAAa,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,QAAQ,KAAK,WAAW;YACnC,CAAC,CAAC,4DAA4D;YAC9D,CAAC,CAAC,QAAQ,KAAK,YAAY;gBACzB,CAAC,CAAC,gGAAgG;gBAClG,CAAC,CAAC,QAAQ,KAAK,WAAW;oBACxB,CAAC,CAAC,uEAAuE;oBACzE,CAAC,CAAC,QAAQ,KAAK,YAAY;wBACzB,CAAC,CAAC,wEAAwE;wBAC1E,CAAC,CAAC,QAAQ,KAAK,QAAQ;4BACrB,CAAC,CAAC,4CAA4C;4BAC9C,CAAC,CAAC,+CAA+C,CAAC;QAC5D,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,uBAAuB;YAC9B,WAAW,EAAE,6EAA6E;YAC1F,IAAI;YACJ,SAAS,EAAE,KAAK;SACjB,CAAC;IACJ,CAAC;IAED,UAAU;IACV,IAAI,UAAU,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,CAAC;QACtC,OAAO;YACL,QAAQ,EAAE,SAAS;YACnB,KAAK,EAAE,mBAAmB;YAC1B,WAAW,EAAE,+DAA+D;YAC5E,IAAI,EAAE,4FAA4F;YAClG,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,uBAAuB;IACvB,IAAI,UAAU,CAAC,GAAG,EAAE,6BAA6B,CAAC,EAAE,CAAC;QACnD,MAAM,YAAY,GAA2B;YAC3C,YAAY,EAAE,8DAA8D;YAC5E,WAAW,EAAE,iDAAiD;YAC9D,YAAY,EAAE,uDAAuD;YACrE,QAAQ,EAAE,4BAA4B;SACvC,CAAC;QACF,OAAO;YACL,QAAQ,EAAE,sBAAsB;YAChC,KAAK,EAAE,sBAAsB;YAC7B,WAAW,EAAE,oCAAoC,QAAQ,IAAI,SAAS,IAAI;YAC1E,IAAI,EAAE,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI,mDAAmD;YACjG,SAAS,EAAE,KAAK;SACjB,CAAC;IACJ,CAAC;IAED,cAAc;IACd,IAAI,UAAU,CAAC,GAAG,EAAE,oBAAoB,CAAC,EAAE,CAAC;QAC1C,OAAO;YACL,QAAQ,EAAE,aAAa;YACvB,KAAK,EAAE,sBAAsB;YAC7B,WAAW,EAAE,wFAAwF;YACrG,IAAI,EAAE,6EAA6E;YACnF,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,UAAU;IACV,OAAO;QACL,QAAQ,EAAE,SAAS;QACnB,KAAK,EAAE,kBAAkB;QACzB,WAAW,EAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG;QAC/D,IAAI,EAAE,wEAAwE;QAC9E,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC"}
+{"version":3,"file":"error-classifier.js","sourceRoot":"","sources":["../../src/eval-server/error-classifier.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2EAA2E;AAC3E,8EAA8E;AAW9E,8EAA8E;AAC9E,sCAAsC;AACtC,8EAA8E;AAE9E,MAAM,mBAAmB,GAAG;IAC1B,cAAc;IACd,oBAAoB;IACpB,KAAK;IACL,UAAU;IACV,kBAAkB;IAClB,aAAa;IACb,WAAW;CACZ,CAAC;AAEF,MAAM,uBAAuB,GAAG;IAC9B,2BAA2B;IAC3B,8BAA8B;IAC9B,mBAAmB;IACnB,mBAAmB;IACnB,mBAAmB;IACnB,4BAA4B;CAC7B,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,WAAW;IACX,OAAO;IACP,KAAK;IACL,KAAK;IACL,eAAe;IACf,YAAY;IACZ,qBAAqB;IACrB,eAAe;CAChB,CAAC;AAEF,MAAM,gBAAgB,GAAG;IACvB,eAAe;IACf,UAAU;IACV,MAAM;IACN,YAAY;IACZ,QAAQ;IACR,UAAU;CACX,CAAC;AAEF,MAAM,6BAA6B,GAAG;IACpC,QAAQ;IACR,YAAY;IACZ,oBAAoB;IACpB,KAAK;IACL,KAAK;IACL,uBAAuB;IACvB,sBAAsB;IACtB,eAAe;IACf,gBAAgB;CACjB,CAAC;AAEF,MAAM,oBAAoB,GAAG;IAC3B,cAAc;IACd,OAAO;IACP,QAAQ;IACR,iBAAiB;IACjB,mBAAmB;CACpB,CAAC;AAEF,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,SAAS,UAAU,CAAC,GAAW,EAAE,QAAkB;IACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAY,EAAE,QAAiB;IAC3D,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;IAEnE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;QAChB,OAAO;YACL,QAAQ,EAAE,SAAS;YACnB,KAAK,EAAE,eAAe;YACtB,WAAW,EAAE,+CAA+C;YAC5D,IAAI,EAAE,wEAAwE;YAC9E,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,aAAa;IACb,IAAI,UAAU,CAAC,GAAG,EAAE,mBAAmB,CAAC,EAAE,CAAC;QACzC,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,KAAK,EAAE,oBAAoB;YAC3B,WAAW,EAAE,mDAAmD;YAChE,IAAI,EAAE,QAAQ,KAAK,YAAY;gBAC7B,CAAC,CAAC,8DAA8D;gBAChE,CAAC,CAAC,+CAA+C;YACnD,SAAS,EAAE,IAAI;YACf,YAAY,EAAE,MAAM;SACrB,CAAC;IACJ,CAAC;IAED,iBAAiB;IACjB,IAAI,UAAU,CAAC,GAAG,EAAE,uBAAuB,CAAC,EAAE,CAAC;QAC7C,OAAO;YACL,QAAQ,EAAE,gBAAgB;YAC1B,KAAK,EAAE,mBAAmB;YAC1B,WAAW,EAAE,uDAAuD;YACpE,IAAI,EAAE,sFAAsF;YAC5F,SAAS,EAAE,KAAK;SACjB,CAAC;IACJ,CAAC;IAED,OAAO;IACP,IAAI,UAAU,CAAC,GAAG,EAAE,aAAa,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,QAAQ,KAAK,WAAW;YACnC,CAAC,CAAC,4DAA4D;YAC9D,CAAC,CAAC,QAAQ,KAAK,YAAY;gBACzB,CAAC,CAAC,gGAAgG;gBAClG,CAAC,CAAC,QAAQ,KAAK,WAAW;oBACxB,CAAC,CAAC,uEAAuE;oBACzE,CAAC,CAAC,QAAQ,KAAK,YAAY;wBACzB,CAAC,CAAC,wEAAwE;wBAC1E,CAAC,CAAC,QAAQ,KAAK,QAAQ;4BACrB,CAAC,CAAC,4CAA4C;4BAC9C,CAAC,CAAC,+CAA+C,CAAC;QAC5D,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,uBAAuB;YAC9B,WAAW,EAAE,6EAA6E;YAC1F,IAAI;YACJ,SAAS,EAAE,KAAK;SACjB,CAAC;IACJ,CAAC;IAED,UAAU;IACV,IAAI,UAAU,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,CAAC;QACtC,OAAO;YACL,QAAQ,EAAE,SAAS;YACnB,KAAK,EAAE,mBAAmB;YAC1B,WAAW,EAAE,+DAA+D;YAC5E,IAAI,EAAE,4FAA4F;YAClG,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,uFAAuF;IACvF,IAAI,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACvE,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;QAC/C,MAAM,QAAQ,GAAG,QAAQ,KAAK,QAAQ;YACpC,CAAC,CAAC,qCAAqC,SAAS,EAAE;YAClD,CAAC,CAAC,cAAc,SAAS,kEAAkE,CAAC;QAC9F,OAAO;YACL,QAAQ,EAAE,iBAAiB;YAC3B,KAAK,EAAE,iBAAiB;YACxB,WAAW,EAAE,wBAAwB,SAAS,qBAAqB;YACnE,IAAI,EAAE,QAAQ;YACd,SAAS,EAAE,KAAK;SACjB,CAAC;IACJ,CAAC;IAED,uBAAuB;IACvB,IAAI,UAAU,CAAC,GAAG,EAAE,6BAA6B,CAAC,EAAE,CAAC;QACnD,MAAM,YAAY,GAA2B;YAC3C,YAAY,EAAE,8DAA8D;YAC5E,WAAW,EAAE,iDAAiD;YAC9D,YAAY,EAAE,uDAAuD;YACrE,QAAQ,EAAE,4BAA4B;SACvC,CAAC;QACF,OAAO;YACL,QAAQ,EAAE,sBAAsB;YAChC,KAAK,EAAE,sBAAsB;YAC7B,WAAW,EAAE,oCAAoC,QAAQ,IAAI,SAAS,IAAI;YAC1E,IAAI,EAAE,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI,mDAAmD;YACjG,SAAS,EAAE,KAAK;SACjB,CAAC;IACJ,CAAC;IAED,cAAc;IACd,IAAI,UAAU,CAAC,GAAG,EAAE,oBAAoB,CAAC,EAAE,CAAC;QAC1C,OAAO;YACL,QAAQ,EAAE,aAAa;YACvB,KAAK,EAAE,sBAAsB;YAC7B,WAAW,EAAE,wFAAwF;YACrG,IAAI,EAAE,6EAA6E;YACnF,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,UAAU;IACV,OAAO;QACL,QAAQ,EAAE,SAAS;QACnB,KAAK,EAAE,kBAAkB;QACzB,WAAW,EAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG;QAC/D,IAAI,EAAE,wEAAwE;QAC9E,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC"}