pi-harness-runtime 1.1.49 → 1.1.71-3

package/packages/jev-judge/src/flaky-detector.ts

@@ -0,0 +1,190 @@
+/**
+ * Flaky Detector - Specialized module for detecting flaky tests
+ *
+ * Uses Jev to analyze test failures and determine if they're flaky.
+ */
+
+import { JevJudge } from "./index.js";
+import type { JevJudgeConfig, E2ETestResult, E2EFlakyAnalysis } from "./types.js";
+
+// Known flaky patterns
+const FLAKY_PATTERNS = {
+  timeout: ["timeout", "timed out", "ETIMEDOUT", "TimeoutError"],
+  network: ["ECONNREFUSED", "ENOTFOUND", "network", "fetch failed"],
+  race_condition: [
+    "race condition",
+    "concurrent",
+    "simultaneous",
+    "out of order",
+  ],
+  ui_state: [
+    "element not visible",
+    "element not interactable",
+    "stale element",
+    "not attached",
+  ],
+};
+
+/**
+ * Flaky Detector - Specialized for detecting flaky test failures
+ */
+export class FlakyDetector {
+  private judge: JevJudge;
+  private threshold: number;
+
+  constructor(config: JevJudgeConfig, threshold = 0.7) {
+    this.judge = new JevJudge(config);
+    this.threshold = threshold;
+  }
+
+  /**
+   * Quick pattern-based flaky detection (without Jev)
+   * Used for fast, synchronous checks
+   */
+  detectPattern(errorMessage?: string): {
+    isFlaky: boolean;
+    pattern: string | null;
+    confidence: "high" | "medium" | "low";
+  } {
+    if (!errorMessage) {
+      return {
+        isFlaky: false,
+        pattern: null,
+        confidence: "low",
+      };
+    }
+
+    const lowerError = errorMessage.toLowerCase();
+
+    for (const [pattern, keywords] of Object.entries(FLAKY_PATTERNS)) {
+      const matches = keywords.filter((kw) => lowerError.includes(kw.toLowerCase()));
+
+      if (matches.length > 0) {
+        return {
+          isFlaky: true,
+          pattern,
+          confidence: matches.length >= 2 ? "high" : "medium",
+        };
+      }
+    }
+
+    return {
+      isFlaky: false,
+      pattern: null,
+      confidence: "low",
+    };
+  }
+
+  /**
+   * AI-powered flaky detection with Jev
+   */
+  async detect(testResult: E2ETestResult): Promise<E2EFlakyAnalysis> {
+    // First, quick pattern check
+    const patternResult = this.detectPattern(testResult.error);
+
+    // If pattern detection is confident, return early
+    if (patternResult.confidence === "high") {
+      return {
+        isFlaky: patternResult.isFlaky,
+        confidence: patternResult.confidence,
+        probability: patternResult.isFlaky ? 0.95 : 0.05,
+        pattern: patternResult.pattern ?? undefined,
+        recommendation: patternResult.isFlaky ? "skip" : "investigate",
+      };
+    }
+
+    // Use Jev for nuanced analysis
+    const state = this.buildState(testResult);
+
+    const result = await this.judge.evaluate(state, {
+      isFlaky: {
+        type: "noul",
+        instructions:
+          "Does this failure match a known flaky pattern? Consider: timeout, network issue, race condition, or intermittent UI state.",
+      },
+      patternType: {
+        type: "choice",
+        instructions: "What is the most likely cause?",
+        options: [
+          "timeout",
+          "network_issue",
+          "race_condition",
+          "ui_intermittent",
+          "resource_contention",
+          "real_bug",
+          "unknown",
+        ],
+      },
+      confidence: {
+        type: "noul",
+        instructions:
+          "Is this assessment based on clear evidence and likely to be correct?",
+      },
+    });
+
+    const isFlakyDecision = result.decisions.isFlaky;
+    const probability = isFlakyDecision.probability ?? 0.5;
+
+    const patternDecision = result.decisions.patternType as {
+      response: { choice: string };
+    };
+
+    return {
+      isFlaky: probability >= this.threshold,
+      confidence: isFlakyDecision.confidence,
+      probability,
+      pattern: patternDecision?.response?.choice,
+      recommendation: this.getRecommendation(probability),
+    };
+  }
+
+  /**
+   * Batch detect flaky tests
+   */
+  async batchDetect(
+    testResults: E2ETestResult[]
+  ): Promise<Map<string, E2EFlakyAnalysis>> {
+    const results = new Map<string, E2EFlakyAnalysis>();
+
+    // Process in batches
+    const BATCH_SIZE = 5;
+    for (let i = 0; i < testResults.length; i += BATCH_SIZE) {
+      const batch = testResults.slice(i, i + BATCH_SIZE);
+
+      const batchResults = await Promise.all(
+        batch.map((test) => this.detect(test))
+      );
+
+      for (let j = 0; j < batch.length; j++) {
+        results.set(batch[j].name, batchResults[j]);
+      }
+    }
+
+    return results;
+  }
+
+  private buildState(test: E2ETestResult): string {
+    const parts: string[] = [`Test: ${test.name}`, `Status: ${test.status}`];
+
+    if (test.error) {
+      parts.push(`Error: ${test.error}`);
+    }
+
+    if (test.duration !== undefined) {
+      parts.push(`Duration: ${test.duration}ms`);
+    }
+
+    if (test.retries !== undefined) {
+      parts.push(`Previous retries: ${test.retries}`);
+    }
+
+    return parts.join("\n");
+  }
+
+  private getRecommendation(probability: number): "retry" | "skip" | "block" | "investigate" {
+    if (probability >= 0.9) return "skip";
+    if (probability >= 0.7) return "retry";
+    if (probability >= 0.4) return "investigate";
+    return "block";
+  }
+}

package/packages/jev-judge/src/index.ts

@@ -0,0 +1,345 @@
+/**
+ * Jev Judge - Main API
+ *
+ * Structured decision model for agent routing, E2E testing, and automated decisions.
+ * Uses TypeSafe Jev via OpenRouter for fast, calibrated, hallucination-free decisions.
+ *
+ * @example
+ * ```typescript
+ * import { JevJudge } from "@pi-harness/jev-judge";
+ *
+ * const judge = new JevJudge({ apiKey: process.env.OPENROUTER_API_KEY! });
+ *
+ * // Ask multiple questions at once
+ * const result = await judge.evaluate(
+ *   "The deploy failed with exit code 1",
+ *   {
+ *     isUrgent: { type: "noul", instructions: "Does this require immediate attention?" },
+ *     severity: {
+ *       type: "choice",
+ *       instructions: "What is the severity level?",
+ *       options: ["critical", "high", "medium", "low"]
+ *     }
+ *   }
+ * );
+ *
+ * console.log(result.decisions.isUrgent.noul); // 0.92
+ * ```
+ */
+
+import type {
+  JevJudgeConfig,
+  JevQuestion,
+  JevQuestions,
+  JevResponse,
+  JevDecision,
+  JevEvaluationResult,
+  JevNoulResponse,
+  JevChoiceResponse,
+  JevScoreResponse,
+  ThresholdDecision,
+} from "./types.js";
+
+// OpenAI SDK (OpenRouter is OpenAI-compatible)
+// @ts-ignore - openai is installed at workspace root
+import OpenAI from "openai";
+
+/**
+ * Default configuration
+ */
+const DEFAULT_CONFIG = {
+  baseUrl: "https://openrouter.ai/api/v1",
+  model: "typesafe/jev-1.13",
+  defaultThreshold: 0.7,
+  timeoutMs: 30000,
+};
+
+/**
+ * JevJudge - Main class for making structured decisions with Jev
+ */
+export class JevJudge {
+  private client: OpenAI;
+  private config: Required<JevJudgeConfig>;
+
+  constructor(config: JevJudgeConfig) {
+    this.config = {
+      ...DEFAULT_CONFIG,
+      ...config,
+    } as Required<JevJudgeConfig>;
+
+    this.client = new OpenAI({
+      apiKey: this.config.apiKey,
+      baseURL: this.config.baseUrl,
+      timeout: this.config.timeoutMs,
+    });
+  }
+
+  /**
+   * Evaluate state against questions
+   *
+   * @param state - The context/situation to evaluate (text or structured)
+   * @param questions - Map of question_id -> question definition
+   * @returns Evaluation result with decisions
+   */
+  async evaluate<T extends string>(
+    state: string | object,
+    questions: JevQuestions<T>
+  ): Promise<JevEvaluationResult<T>> {
+    const startTime = Date.now();
+
+    // Convert state to string if object
+    const stateStr = typeof state === "string" ? state : JSON.stringify(state);
+
+    // Build questions array for API
+    const questionsArray = Object.entries(questions).map(([id, q]) => {
+      const questionObj = q as JevQuestion;
+      const entry: Record<string, unknown> = { id };
+      if (questionObj.type === "noul") {
+        entry.type = "noul";
+        entry.instructions = questionObj.instructions;
+      } else if (questionObj.type === "choice") {
+        entry.type = "choice";
+        entry.instructions = questionObj.instructions;
+        entry.options = questionObj.options;
+      } else if (questionObj.type === "score") {
+        entry.type = "score";
+        entry.instructions = questionObj.instructions;
+        if (questionObj.min !== undefined) entry.min = questionObj.min;
+        if (questionObj.max !== undefined) entry.max = questionObj.max;
+      }
+      return entry;
+    });
+
+    // Call Jev via OpenRouter
+    const response = await this.client.chat.completions.create({
+      model: this.config.model,
+      messages: [
+        {
+          role: "system",
+          content: `You are a decision engine. Evaluate the following state and answer the questions.
+Return your answers in the exact JSON format specified.`,
+        },
+        {
+          role: "user",
+          content: `State: ${stateStr}
+
+Questions:
+${JSON.stringify(questionsArray, null, 2)}
+
+Respond with a JSON object mapping each question ID to its answer.`,
+        },
+      ],
+      response_format: {
+        type: "json_object",
+        schema: this.buildResponseSchema(questions),
+      },
+      temperature: 0.1, // Low temperature for consistent decisions
+    });
+
+    const latencyMs = Date.now() - startTime;
+
+    // Parse response
+    const content = response.choices[0]?.message?.content;
+    if (!content) {
+      throw new Error("No response from Jev");
+    }
+
+    let parsed: Record<string, unknown>;
+    try {
+      parsed = JSON.parse(content);
+    } catch {
+      throw new Error("Invalid JSON response from Jev");
+    }
+
+    // Build decisions map
+    const decisions = {} as Record<T, JevDecision<T>>;
+
+    for (const [questionId, question] of Object.entries(questions)) {
+      const questionData = question as JevQuestion;
+      const responseData = (parsed as Record<string, unknown>)[questionId] as
+        | JevResponse
+        | undefined;
+
+      if (!responseData) {
+        continue;
+      }
+
+      const decision = this.buildDecision(
+        questionId,
+        questionData,
+        responseData
+      );
+      (decisions as Record<string, JevDecision>)[questionId] = decision;
+    }
+
+    return {
+      decisions,
+      state: stateStr,
+      latencyMs,
+      model: this.config.model,
+    };
+  }
+
+  /**
+   * Quick noul (yes/no) evaluation
+   */
+  async noul(
+    state: string | object,
+    question: string
+  ): Promise<{ noul: number; confidence: "high" | "medium" | "low" }> {
+    const result = await this.evaluate(state, {
+      answer: { type: "noul", instructions: question },
+    });
+
+    const decision = result.decisions.answer as JevDecision<"answer">;
+    return {
+      noul: (decision.response as JevNoulResponse).noul,
+      confidence: decision.confidence,
+    };
+  }
+
+  /**
+   * Quick choice evaluation
+   */
+  async choice<T extends string>(
+    state: string | object,
+    question: string,
+    options: T[]
+  ): Promise<{ choice: T; confidence: number }> {
+    const result = await this.evaluate(state, {
+      answer: { type: "choice", instructions: question, options },
+    });
+
+    const decision = result.decisions.answer as JevDecision<"answer">;
+    return {
+      choice: (decision.response as JevChoiceResponse<T>).choice,
+      confidence: (decision.response as JevChoiceResponse).confidence,
+    };
+  }
+
+  /**
+   * Make threshold-based decision
+   */
+  thresholdDecision(
+    probability: number,
+    threshold: number = this.config.defaultThreshold
+  ): ThresholdDecision {
+    const confidence = this.getConfidence(probability, threshold);
+
+    return {
+      shouldAct: probability >= threshold,
+      probability,
+      confidence,
+      action: probability >= 0.9 ? "proceed" : probability >= 0.5 ? "review" : "block",
+    };
+  }
+
+  /**
+   * Batch evaluate multiple states
+   */
+  async batchEvaluate<T extends string>(
+    states: (string | object)[],
+    questions: JevQuestions<T>,
+    options: { concurrency?: number } = {}
+  ): Promise<JevEvaluationResult<T>[]> {
+    const { concurrency = 3 } = options;
+
+    const results: JevEvaluationResult<T>[] = [];
+    const batches: (string | object)[] = [];
+
+    // Split into batches
+    for (let i = 0; i < states.length; i += concurrency) {
+      batches.push(...states.slice(i, i + concurrency));
+    }
+
+    // Process batches
+    for (const batch of batches) {
+      const result = await this.evaluate(batch, questions);
+      results.push(result);
+    }
+
+    return results;
+  }
+
+  // Private helpers
+
+  private buildResponseSchema(questions: JevQuestions): Record<string, unknown> {
+    const schema: Record<string, unknown> = {};
+
+    for (const [id, question] of Object.entries(questions)) {
+      switch ((question as JevQuestion).type) {
+        case "noul":
+          schema[id] = { type: "object", properties: {
+            noul: { type: "number", minimum: 0, maximum: 1 }
+          }, required: ["noul"] };
+          break;
+        case "choice":
+          schema[id] = { type: "object", properties: {
+            choice: { type: "string", enum: (question as { options: string[] }).options },
+            confidence: { type: "number", minimum: 0, maximum: 1 }
+          }, required: ["choice"] };
+          break;
+        case "score":
+          schema[id] = { type: "object", properties: {
+            score: { type: "number" },
+            confidence: { type: "number", minimum: 0, maximum: 1 }
+          }, required: ["score"] };
+          break;
+      }
+    }
+
+    return { type: "object", properties: schema, additionalProperties: false };
+  }
+
+  private buildDecision<T extends string>(
+    questionId: T,
+    question: JevQuestion,
+    response: JevResponse
+  ): JevDecision<T> {
+    let probability: number;
+    let confidence: "high" | "medium" | "low";
+
+    switch (response.type) {
+      case "noul":
+        probability = response.noul;
+        break;
+      case "choice":
+        probability = response.confidence;
+        break;
+      case "score":
+        // Normalize score to 0-1
+        const max = question.type === "score" ? (question.max ?? 100) : 100;
+        const min = question.type === "score" ? (question.min ?? 0) : 0;
+        probability = (response.score - min) / (max - min);
+        break;
+    }
+
+    confidence = this.getConfidence(probability, this.config.defaultThreshold);
+
+    return {
+      questionId,
+      question,
+      response,
+      probability,
+      confidence,
+    };
+  }
+
+  private getConfidence(
+    probability: number,
+    threshold: number
+  ): "high" | "medium" | "low" {
+    const distance = Math.abs(probability - threshold);
+    if (distance > 0.3) return "high";
+    if (distance > 0.15) return "medium";
+    return "low";
+  }
+}
+
+// Export types
+export * from "./types.js";
+
+// Export submodules
+export { E2EJudge } from "./e2e-judge.js";
+export { FlakyDetector } from "./flaky-detector.js";
+export { DecisionEngine } from "./decision-engine.js";

package/packages/jev-judge/src/types.ts

@@ -0,0 +1,131 @@
+/**
+ * Jev Judge - Type definitions
+ *
+ * Jev is a System One model from TypeSafe AI that makes fast, structured decisions
+ * with calibrated probabilities. Available via OpenRouter: typesafe/jev-1.13
+ */
+
+// Question types supported by Jev
+export type JevQuestionType = "noul" | "choice" | "score";
+
+// Noul: Yes/No question (returns probability 0-1)
+export interface JevNoulQuestion {
+  type: "noul";
+  instructions: string;
+}
+
+// Choice: Select from options (returns selected option)
+export interface JevChoiceQuestion<T extends string = string> {
+  type: "choice";
+  instructions: string;
+  options: T[];
+}
+
+// Score: Rate something 0-100
+export interface JevScoreQuestion {
+  type: "score";
+  instructions: string;
+  min?: number;
+  max?: number;
+}
+
+// Any question type
+export type JevQuestion = JevNoulQuestion | JevChoiceQuestion | JevScoreQuestion;
+
+// Questions map (question_id -> question)
+export type JevQuestions<T extends string = string> = Partial<
+  Record<T, JevQuestion>
+>;
+
+// Response types
+export interface JevNoulResponse {
+  type: "noul";
+  noul: number; // 0-1 probability
+}
+
+export interface JevChoiceResponse<T extends string = string> {
+  type: "choice";
+  choice: T;
+  confidence: number; // 0-1
+}
+
+export interface JevScoreResponse {
+  type: "score";
+  score: number;
+  confidence: number; // 0-1
+}
+
+// Any response type
+export type JevResponse = JevNoulResponse | JevChoiceResponse | JevScoreResponse;
+
+// Response map
+export type JevResponses<T extends string = string> = Partial<Record<T, JevResponse>>;
+
+// Decision result with metadata
+export interface JevDecision<T extends string = string> {
+  questionId: T;
+  question: JevQuestion;
+  response: JevResponse;
+  probability?: number; // Extracted for easy access (noul = noul value, choice = confidence)
+  confidence: "high" | "medium" | "low";
+}
+
+// Full evaluation result
+export interface JevEvaluationResult<T extends string = string> {
+  decisions: Record<T, JevDecision<T>>;
+  state: string;
+  latencyMs: number;
+  model: string;
+}
+
+// Threshold-based action decision
+export interface ThresholdDecision {
+  shouldAct: boolean;
+  probability: number;
+  confidence: "high" | "medium" | "low";
+  action: "proceed" | "review" | "block";
+}
+
+// Config
+export interface JevJudgeConfig {
+  apiKey: string;
+  baseUrl?: string; // Defaults to OpenRouter
+  model?: string; // Defaults to typesafe/jev-1.13
+  defaultThreshold?: number; // Default 0.7
+  timeoutMs?: number;
+}
+
+// E2E Test Judge types
+export interface E2ETestResult {
+  name: string;
+  status: "passed" | "failed" | "skipped" | "pending";
+  duration?: number; // ms
+  error?: string;
+  screenshot?: string;
+  consoleLogs?: string[];
+  networkLogs?: string[];
+  retries?: number;
+}
+
+export interface E2EFlakyAnalysis {
+  isFlaky: boolean;
+  confidence: "high" | "medium" | "low";
+  probability: number;
+  pattern?: string;
+  recommendation: "retry" | "skip" | "block" | "investigate";
+}
+
+export interface E2EBuildDecision {
+  shouldBlock: boolean;
+  probability: number;
+  confidence: "high" | "medium" | "low";
+  reason: string;
+  affectedTests: string[];
+}
+
+export interface E2ERetryDecision {
+  shouldRetry: boolean;
+  maxAttempts: number;
+  probability: number;
+  delayMs?: number;
+}

package/packages/knowledge-graph/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/knowledge-graph",
-  "version": "1.1.49",
+  "version": "1.1.71-3",
   "description": "Knowledge Graph extraction and provenance linking for SKILL.md files",
   "type": "module",
   "main": "./dist/index.js",

package/packages/knowledge-retrieval/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/knowledge-retrieval",
-  "version": "1.1.49",
+  "version": "1.1.71-3",
   "description": "Knowledge retrieval via TencentDB-Agent-Memory MCP (RFC-0105)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/laravel-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/laravel-plugin",
-  "version": "1.1.49",
+  "version": "1.1.71-3",
   "description": "Laravel framework analysis — artisan commands, Eloquent models, Blade",
   "type": "module",
   "main": "./dist/index.js",

package/packages/learning-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/learning-engine",
-  "version": "1.1.49",
+  "version": "1.1.71-3",
   "description": "Learning engine for extracting lessons from runtime execution (RFC-0058)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/mcp-adapter/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/mcp-adapter",
-  "version": "1.1.49",
+  "version": "1.1.71-3",
   "type": "module",
   "description": "MCP server adapter for pi-harness — exposes harness capabilities as MCP tools/resources/prompts (RFC-0068)",
   "main": "dist/index.js",