playwright-intelligence-engine 0.0.0-stage → 2.0.0

package/src/integrations/sarif-report.ts

@@ -0,0 +1,110 @@
+import type { CiEvidenceReport } from './playwright-report-adapter.js';
+
+export interface SarifReport {
+  version: '2.1.0';
+  $schema: 'https://json.schemastore.org/sarif-2.1.0.json';
+  runs: Array<{
+    tool: { driver: { name: string; version: string; informationUri: string; rules: SarifRule[] } };
+    results: SarifResult[];
+  }>;
+}
+
+interface SarifRule {
+  id: string;
+  name: string;
+  shortDescription: { text: string };
+  defaultConfiguration: { level: 'error' | 'warning' | 'note' };
+}
+
+interface SarifResult {
+  ruleId: string;
+  level: 'error' | 'warning' | 'note';
+  message: { text: string };
+  properties: Record<string, string | number | boolean>;
+  locations?: Array<{ physicalLocation: { artifactLocation: { uri: string }; region?: { startLine: number } } }>;
+}
+
+function levelForSeverity(severity: string): SarifResult['level'] {
+  if (severity === 'critical' || severity === 'high') return 'error';
+  if (severity === 'medium') return 'warning';
+  return 'note';
+}
+
+function ruleForCategory(category: string, level: SarifResult['level']): SarifRule {
+  return {
+    id: `PIE-${category}`,
+    name: category,
+    shortDescription: { text: `PIE failure category ${category}` },
+    defaultConfiguration: { level },
+  };
+}
+
+/** Converts normalized PIE CI evidence into a standard SARIF 2.1.0 artifact. */
+export function buildSarifReport(report: CiEvidenceReport, toolVersion = '0.1.0'): SarifReport {
+  const rules = new Map<string, SarifRule>();
+  const results: SarifResult[] = [];
+
+  for (const cluster of report.clusters) {
+    const level = levelForSeverity(cluster.severity);
+    const category = cluster.fingerprint.category;
+    const ruleId = `PIE-${category}`;
+    if (!rules.has(ruleId)) rules.set(ruleId, ruleForCategory(category, level));
+    const representative = cluster.representative;
+    const result: SarifResult = {
+      ruleId,
+      level,
+      message: { text: `${category}: ${representative.errorMessage ?? 'Failure cluster detected'}` },
+      properties: {
+        clusterId: cluster.clusterId,
+        count: cluster.count,
+        severity: cluster.severity,
+        affectedTests: cluster.affectedTests.length,
+        affectedRepositories: cluster.affectedRepositories.length,
+      },
+    };
+    if (representative.sourceFile) {
+      result.locations = [{
+        physicalLocation: {
+          artifactLocation: { uri: representative.sourceFile.replace(/\\/g, '/') },
+        },
+      }];
+    }
+    results.push(result);
+  }
+
+  const gateRuleId = 'PIE-QUALITY-GATE';
+  const gateLevel: SarifResult['level'] = report.qualityGate.status === 'FAIL' ? 'error' : report.qualityGate.status === 'ABSTAIN' ? 'warning' : 'note';
+  rules.set(gateRuleId, {
+    id: gateRuleId,
+    name: 'Quality gate',
+    shortDescription: { text: 'PIE deterministic CI quality gate' },
+    defaultConfiguration: { level: gateLevel },
+  });
+  results.push({
+    ruleId: gateRuleId,
+    level: gateLevel,
+    message: { text: `PIE quality gate: ${report.qualityGate.status}${report.qualityGate.reasons.length ? ` — ${report.qualityGate.reasons.join('; ')}` : ''}` },
+    properties: {
+      status: report.qualityGate.status,
+      testsObserved: report.runs.length,
+      failures: report.failures.length,
+      flakyTests: report.stability.filter((test) => test.flaky).length,
+    },
+  });
+
+  return {
+    version: '2.1.0',
+    $schema: 'https://json.schemastore.org/sarif-2.1.0.json',
+    runs: [{
+      tool: {
+        driver: {
+          name: 'PIE',
+          version: toolVersion,
+          informationUri: 'https://github.com/playwright-intelligence-engine',
+          rules: [...rules.values()],
+        },
+      },
+      results,
+    }],
+  };
+}

package/src/intelligence/autonomous-repair-workflow.ts

@@ -0,0 +1,139 @@
+import { LlmRepairProposal } from './llm-proposal.js';
+import { TraceProposalWorkflowInput, TraceProposalWorkflowResult, runTraceProposalWorkflow } from './trace-proposal-workflow.js';
+import { runVerifiedLlmRepairWorkflow } from './verified-llm-repair-workflow.js';
+import { RepairWorkflowResult } from '../repair/repair-workflow.js';
+import { ApprovalService } from '../repair/approval-service.js';
+import { WorkspacePromoter, PromotionResult } from '../repair/workspace-promoter.js';
+
+export interface AutonomousRepairWorkflowInput extends TraceProposalWorkflowInput {
+  execute?: boolean | undefined;
+  testFilePath?: string | undefined;
+  pomFilePath?: string | undefined;
+  className?: string | undefined;
+  memberName?: string | undefined;
+  configPath?: string | undefined;
+  worktreeId?: string | undefined;
+  autoPromote?: boolean | undefined;
+}
+
+export type AutonomousRepairStatus =
+  | 'ABSTAINED'
+  | 'MODEL_NOT_CONFIGURED'
+  | 'PROPOSAL_REJECTED'
+  | 'PROPOSAL_READY'
+  | 'INPUT_REQUIRED'
+  | 'VERIFIED_READY_FOR_APPROVAL'
+  | 'PROMOTED'
+  | 'VERIFICATION_FAILED';
+
+export interface AutonomousRepairWorkflowResult {
+  status: AutonomousRepairStatus;
+  proposalRun: TraceProposalWorkflowResult;
+  proposal?: LlmRepairProposal | undefined;
+  verification?: RepairWorkflowResult | undefined;
+  promotion?: PromotionResult | undefined;
+  reason?: string | undefined;
+}
+
+/**
+ * Connects PIE's bounded reasoning loop to its existing deterministic repair
+ * verifier. The model can propose one allowlisted operation, but only the
+ * deterministic workflow can apply it in a disposable worktree and decide
+ * whether it is eligible for approval.
+ */
+export async function runAutonomousRepairWorkflow(
+  input: AutonomousRepairWorkflowInput,
+): Promise<AutonomousRepairWorkflowResult> {
+  const proposalRun = await runTraceProposalWorkflow(input);
+  const agent = proposalRun.agent;
+
+  if (!agent.proposal) {
+    const reason = agent.errors.join('; ') || 'The agent did not produce a repair proposal.';
+    const status: AutonomousRepairStatus = agent.state === 'MODEL_NOT_CONFIGURED'
+      ? 'MODEL_NOT_CONFIGURED'
+      : agent.state === 'PROPOSAL_REJECTED'
+        ? 'PROPOSAL_REJECTED'
+        : 'ABSTAINED';
+    return { status, proposalRun, reason };
+  }
+
+  if (agent.proposal.decision === 'ABSTAIN') {
+    return {
+      status: 'ABSTAINED',
+      proposalRun,
+      proposal: agent.proposal,
+      reason: agent.proposal.reasoningSummary,
+    };
+  }
+
+  if (input.execute !== true && input.autoPromote !== true) {
+    return {
+      status: 'PROPOSAL_READY',
+      proposalRun,
+      proposal: agent.proposal,
+      reason: 'Proposal validated. Execution was not requested.',
+    };
+  }
+
+  let verification: RepairWorkflowResult;
+  try {
+    verification = await runVerifiedLlmRepairWorkflow({
+      tracePath: input.tracePath,
+      workspaceRoot: input.workspaceRoot,
+      errorContextPath: input.errorContextPath,
+      testFilePath: input.testFilePath,
+      pomFilePath: input.pomFilePath,
+      className: input.className,
+      memberName: input.memberName,
+      configPath: input.configPath,
+      worktreeId: input.worktreeId,
+      execute: true,
+      proposal: agent.proposal,
+    });
+  } catch (error) {
+    return {
+      status: 'PROPOSAL_REJECTED',
+      proposalRun,
+      proposal: agent.proposal,
+      reason: error instanceof Error ? error.message : String(error),
+    };
+  }
+
+  const status: AutonomousRepairStatus = verification.status === 'VERIFIED_READY_FOR_APPROVAL'
+    ? 'VERIFIED_READY_FOR_APPROVAL'
+    : verification.status === 'INPUT_REQUIRED'
+      ? 'INPUT_REQUIRED'
+      : verification.status === 'ABSTAINED'
+        ? 'ABSTAINED'
+        : 'VERIFICATION_FAILED';
+
+  if (status === 'VERIFIED_READY_FOR_APPROVAL' && input.autoPromote === true) {
+    const staged = verification.stagedProposal;
+    if (!staged || !verification.proposal) {
+      return { status: 'VERIFICATION_FAILED', proposalRun, proposal: agent.proposal, verification, reason: 'Verified repair did not produce a promotable staged proposal.' };
+    }
+    try {
+      const token = ApprovalService.getInstance().authorizeProposal({ proposalId: staged.proposalId });
+      const promotion = await WorkspacePromoter.promote({
+        targetFilePath: token.targetFilePath,
+        workspaceRoot: input.workspaceRoot ?? process.cwd(),
+        patchedContent: verification.proposal.proposedPatchedSource,
+        approvalTokenPayload: token,
+      });
+      if (!promotion.success) {
+        return { status: 'VERIFICATION_FAILED', proposalRun, proposal: agent.proposal, verification, promotion, reason: promotion.error ?? 'Promotion failed.' };
+      }
+      return { status: 'PROMOTED', proposalRun, proposal: agent.proposal, verification, promotion, reason: 'Verified repair was promoted under explicit auto-promote policy.' };
+    } catch (error) {
+      return { status: 'VERIFICATION_FAILED', proposalRun, proposal: agent.proposal, verification, reason: error instanceof Error ? error.message : String(error) };
+    }
+  }
+
+  return {
+    status,
+    proposalRun,
+    proposal: agent.proposal,
+    verification,
+    reason: verification.reason,
+  };
+}

package/src/intelligence/business-intent.ts

@@ -0,0 +1,163 @@
+import { BusinessOracleStatus } from '../core/types/repair-evidence.js';
+
+export interface BusinessIntent {
+  id: string;
+  title?: string | undefined;
+  actions: string[];
+  entities: string[];
+  expectedOutcomes: string[];
+  assertions: string[];
+  risk: 'critical' | 'high' | 'medium' | 'low' | 'unknown';
+  evidenceSources: string[];
+}
+
+export interface BusinessOracleContext {
+  intent: BusinessIntent;
+  originalAssertions?: string[] | undefined;
+  currentAssertions?: string[] | undefined;
+  observedStates?: Record<string, unknown> | undefined;
+  executionPassed: boolean;
+}
+
+export interface BusinessOracleResult {
+  name: string;
+  status: BusinessOracleStatus;
+  reason: string;
+  evidence: string[];
+}
+
+export interface BusinessOracle {
+  name: string;
+  evaluate(context: BusinessOracleContext): BusinessOracleResult;
+}
+
+export interface BuiltInOracleEvaluation {
+  status: BusinessOracleStatus;
+  results: BusinessOracleResult[];
+}
+
+const ACTION_WORDS = ['login', 'logout', 'search', 'create', 'submit', 'save', 'update', 'delete', 'approve', 'reject', 'checkout', 'payment', 'book', 'contact', 'upload', 'download'];
+const CRITICAL_WORDS = ['payment', 'checkout', 'delete', 'approve', 'permission', 'authentication', 'login', 'order'];
+
+function words(value: string): string[] {
+  return [...new Set(value.toLowerCase().split(/[^a-z0-9]+/).filter((word) => word.length >= 3))];
+}
+
+function riskFor(text: string): BusinessIntent['risk'] {
+  const normalized = text.toLowerCase();
+  if (CRITICAL_WORDS.some((word) => normalized.includes(word))) return 'critical';
+  if (['account', 'profile', 'customer', 'booking'].some((word) => normalized.includes(word))) return 'high';
+  if (normalized.length > 0) return 'medium';
+  return 'unknown';
+}
+
+/** Extracts a conservative intent model from test titles and source text. */
+export function extractBusinessIntent(input: { id: string; title?: string; sourceText: string; sourcePath?: string }): BusinessIntent {
+  const title = input.title?.trim();
+  const combined = `${title ?? ''}\n${input.sourceText}`;
+  const lower = combined.toLowerCase();
+  const actions = ACTION_WORDS.filter((action) => lower.includes(action));
+  const assertions = [...input.sourceText.matchAll(/expect(?:\.soft|\.poll)?\([^\n]{0,240}/g)].map((match) => match[0]!.trim()).slice(0, 50);
+  const expectedOutcomes = [...input.sourceText.matchAll(/(?:toBe|toEqual|toContainText|toHaveText|toBeVisible|toBeHidden|toHaveURL)\s*\([^\n]{0,160}/g)].map((match) => match[0]!.trim()).slice(0, 50);
+  const titleWords = words(title ?? '');
+  const entities = [...new Set(titleWords.filter((word) => !ACTION_WORDS.includes(word)))].slice(0, 20);
+  return {
+    id: input.id,
+    title,
+    actions,
+    entities,
+    expectedOutcomes,
+    assertions,
+    risk: riskFor(combined),
+    evidenceSources: [
+      ...(title ? ['test-title'] : []),
+      ...(assertions.length > 0 ? ['test-assertions'] : []),
+      ...(input.sourcePath ? [input.sourcePath] : []),
+    ],
+  };
+}
+
+export class BusinessOracleRegistry {
+  private readonly oracles: BusinessOracle[] = [];
+
+  public register(oracle: BusinessOracle): void {
+    if (this.oracles.some((existing) => existing.name === oracle.name)) {
+      throw new Error(`Business oracle '${oracle.name}' is already registered`);
+    }
+    this.oracles.push(oracle);
+  }
+
+  public evaluate(context: BusinessOracleContext): BusinessOracleResult[] {
+    return this.oracles.map((oracle) => oracle.evaluate(context));
+  }
+}
+
+export const AssertionPreservationOracle: BusinessOracle = {
+  name: 'assertion-preservation',
+  evaluate(context): BusinessOracleResult {
+    if (!context.originalAssertions || !context.currentAssertions) {
+      return {
+        name: 'assertion-preservation',
+        status: 'unavailable',
+        reason: 'Original and current assertion evidence were not supplied',
+        evidence: [],
+      };
+    }
+    const original = JSON.stringify(context.originalAssertions);
+    const current = JSON.stringify(context.currentAssertions);
+    const preserved = original === current;
+    return {
+      name: 'assertion-preservation',
+      status: preserved ? 'verified' : 'contradicted',
+      reason: preserved ? 'Business assertions are unchanged' : 'Business assertion evidence changed',
+      evidence: [`Original assertions: ${context.originalAssertions.length}`, `Current assertions: ${context.currentAssertions.length}`],
+    };
+  },
+};
+
+export const ExecutionOracle: BusinessOracle = {
+  name: 'execution-success',
+  evaluate(context): BusinessOracleResult {
+    return {
+      name: 'execution-success',
+      status: context.executionPassed ? 'verified' : 'contradicted',
+      reason: context.executionPassed ? 'Target execution passed' : 'Target execution did not pass',
+      evidence: [`Intent risk: ${context.intent.risk}`, `Expected outcomes: ${context.intent.expectedOutcomes.length}`],
+    };
+  },
+};
+
+/**
+ * Evaluates the evidence PIE can prove locally for an isolated repair.
+ * This deliberately does not claim domain truth such as whether a payment
+ * was actually captured; domain-specific adapters can add that later.
+ */
+export function evaluateBuiltInBusinessOracles(input: {
+  intent: BusinessIntent;
+  originalSourceText?: string | undefined;
+  currentSourceText?: string | undefined;
+  executionPassed: boolean;
+}): BuiltInOracleEvaluation {
+  const registry = new BusinessOracleRegistry();
+  registry.register(AssertionPreservationOracle);
+  registry.register(ExecutionOracle);
+
+  const results = registry.evaluate({
+    intent: input.intent,
+    originalAssertions: input.originalSourceText === undefined
+      ? undefined
+      : extractBusinessIntent({ id: `${input.intent.id}:original`, sourceText: input.originalSourceText }).assertions,
+    currentAssertions: input.currentSourceText === undefined
+      ? undefined
+      : extractBusinessIntent({ id: `${input.intent.id}:current`, sourceText: input.currentSourceText }).assertions,
+    executionPassed: input.executionPassed,
+  });
+
+  const status: BusinessOracleStatus = results.some((result) => result.status === 'contradicted')
+    ? 'contradicted'
+    : results.length > 0 && results.every((result) => result.status === 'verified')
+      ? 'verified'
+      : 'unavailable';
+
+  return { status, results };
+}