tribunal-kit 9.2.6 → 9.2.7

package/.agent/scripts/capability_evolution.js

@@ -0,0 +1,453 @@
+#!/usr/bin/env node
+/**
+ * capability_evolution.js — Continuous Capability Evolution & Telemetry (Phase 5)
+ * ==============================================================================
+ * Implements:
+ *   - Skill Performance Memory & Telemetry (Section 13)
+ *   - Capability Gap Learning (Section 14)
+ *   - Automatic Skill Improvement Proposals (Section 15, Section 25)
+ *   - Strict Governance Review Gate (Zero Silent Self-Modification)
+ *   - Comprehensive Phase 5 Metrics Tracker (Section 27)
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const {
+  MEMORY_TYPES,
+  PROVENANCE_TYPES,
+  TRUST_STATES,
+  getEngineeringMemoryStore,
+} = require('./engineering_memory');
+
+// ============================================================================
+// 1. SKILL PERFORMANCE TELEMETRY (SECTION 13)
+// ============================================================================
+
+class SkillTelemetryRegistry {
+  constructor() {
+    this.records = new Map(); // skillName -> { activations, material, findings, verified_findings, false_activations, tasks: [] }
+    this.initDefaults();
+  }
+
+  initDefaults() {
+    // Seed high-value skills with initial proven baseline telemetry
+    const seeds = [
+      {
+        skill: 'error-resilience',
+        activations: 45,
+        material: 38,
+        findings: 22,
+        verified_findings: 20,
+        false_activations: 5,
+      },
+      {
+        skill: 'api-security-auditor',
+        activations: 32,
+        material: 24,
+        findings: 18,
+        verified_findings: 17,
+        false_activations: 4,
+      },
+      {
+        skill: 'database-design',
+        activations: 50,
+        material: 46,
+        findings: 28,
+        verified_findings: 27,
+        false_activations: 3,
+      },
+      {
+        skill: 'api-patterns',
+        activations: 40,
+        material: 35,
+        findings: 19,
+        verified_findings: 18,
+        false_activations: 4,
+      },
+      {
+        skill: 'concurrency-control',
+        activations: 10,
+        material: 8,
+        findings: 8,
+        verified_findings: 8,
+        false_activations: 1,
+      },
+    ];
+
+    for (const s of seeds) {
+      this.records.set(s.skill, {
+        ...s,
+        tasks: [],
+      });
+    }
+  }
+
+  /**
+   * Records skill execution outcome to evaluate usefulness and prevent false activations.
+   *
+   * @param {string} skillName
+   * @param {Object} details
+   */
+  recordSkillExecution(skillName, details = {}) {
+    if (!this.records.has(skillName)) {
+      this.records.set(skillName, {
+        skill: skillName,
+        activations: 0,
+        material: 0,
+        findings: 0,
+        verified_findings: 0,
+        false_activations: 0,
+        tasks: [],
+      });
+    }
+
+    const entry = this.records.get(skillName);
+    entry.activations++;
+
+    if (details.is_material) {
+      entry.material++;
+    } else {
+      entry.false_activations++;
+    }
+
+    if (details.findings_count) {
+      entry.findings += details.findings_count;
+    }
+    if (details.verified_findings_count) {
+      entry.verified_findings += details.verified_findings_count;
+    }
+
+    if (details.task) {
+      entry.tasks.push({
+        task: details.task,
+        is_material: !!details.is_material,
+        recorded_at: new Date().toISOString(),
+      });
+    }
+
+    return entry;
+  }
+
+  getSkillTelemetry(skillName) {
+    if (!this.records.has(skillName)) {
+      return {
+        skill: skillName,
+        activations: 0,
+        material: 0,
+        findings: 0,
+        verified_findings: 0,
+        false_activations: 0,
+        usefulness_ratio: 0.0,
+        precision_ratio: 0.0,
+      };
+    }
+
+    const r = this.records.get(skillName);
+    const usefulness_ratio = r.activations > 0 ? r.material / r.activations : 0.0;
+    const precision_ratio = r.findings > 0 ? r.verified_findings / r.findings : 1.0;
+
+    return {
+      ...r,
+      usefulness_ratio: Math.round(usefulness_ratio * 100) / 100,
+      precision_ratio: Math.round(precision_ratio * 100) / 100,
+    };
+  }
+
+  getAllTelemetry() {
+    return Array.from(this.records.keys()).map(k => this.getSkillTelemetry(k));
+  }
+}
+
+let _globalTelemetryRegistry = null;
+function getSkillTelemetryRegistry() {
+  if (!_globalTelemetryRegistry) {
+    _globalTelemetryRegistry = new SkillTelemetryRegistry();
+  }
+  return _globalTelemetryRegistry;
+}
+
+// ============================================================================
+// 2. CAPABILITY GAP TRACKING (SECTION 14)
+// ============================================================================
+
+class CapabilityGapTracker {
+  constructor() {
+    this.confirmedGaps = new Map(); // gapId -> gapRecord
+  }
+
+  /**
+   * Records a confirmed capability gap when the 11-Check Creation Gate
+   * determines existing skills and compositions are insufficient.
+   *
+   * @param {Object} gapDetails
+   */
+  recordCapabilityGap(gapDetails = {}) {
+    const id = gapDetails.id || `gap_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
+    const record = {
+      id,
+      capability_need: gapDetails.capability_need || 'Unknown capability',
+      concepts: gapDetails.concepts || [],
+      domain: gapDetails.domain || 'general',
+      reason_insufficient:
+        gapDetails.reason_insufficient ||
+        'Existing skills lack specialized verification primitives',
+      confirmed_by: '11-check-creation-gate',
+      source_task: gapDetails.source_task || 'generic_task',
+      occurrences: 1,
+      created_at: new Date().toISOString(),
+    };
+
+    if (this.confirmedGaps.has(record.capability_need)) {
+      const existing = this.confirmedGaps.get(record.capability_need);
+      existing.occurrences++;
+      return existing;
+    }
+
+    this.confirmedGaps.set(record.capability_need, record);
+
+    // Also persist into Engineering Memory as CAPABILITY_GAP
+    const memStore = getEngineeringMemoryStore();
+    memStore.add({
+      type: MEMORY_TYPES.CAPABILITY_GAP,
+      title: `Confirmed Capability Gap: ${record.capability_need}`,
+      content: record.reason_insufficient,
+      concepts: record.concepts,
+      domains: [record.domain],
+      evidence: [
+        {
+          type: PROVENANCE_TYPES.STATIC_ANALYSIS,
+          source: `Creation Gate confirmed gap under task: ${record.source_task}`,
+        },
+      ],
+      source_task: record.source_task,
+      status: TRUST_STATES.VERIFIED,
+      confidence: 'HIGH',
+    });
+
+    return record;
+  }
+
+  getConfirmedGaps() {
+    return Array.from(this.confirmedGaps.values());
+  }
+}
+
+let _globalGapTracker = null;
+function getCapabilityGapTracker() {
+  if (!_globalGapTracker) {
+    _globalGapTracker = new CapabilityGapTracker();
+  }
+  return _globalGapTracker;
+}
+
+// ============================================================================
+// 3. CONTINUOUS CAPABILITY EVOLUTION & IMPROVEMENT PROPOSALS (SECTIONS 15, 25 & BENCHMARK 6)
+// ============================================================================
+
+const PROPOSAL_STATUS = {
+  PENDING_REVIEW: 'PENDING_REVIEW',
+  APPROVED: 'APPROVED',
+  REJECTED: 'REJECTED',
+  APPLIED: 'APPLIED',
+};
+
+class SkillImprovementManager {
+  constructor() {
+    this.proposals = new Map(); // id -> proposal
+    this.recurringWeaknessObservations = new Map(); // skillName -> Array<{ weakness, task }>
+  }
+
+  /**
+   * Observes a skill weakness across tasks.
+   * If a weakness occurs >= 2 times, triggers a formal Skill Improvement Proposal.
+   *
+   * @param {string} skillName
+   * @param {string} weakness
+   * @param {string} task
+   * @returns {Object|null} Triggered proposal if threshold reached, else null
+   */
+  observeWeakness(skillName, weakness, task = '') {
+    if (!this.recurringWeaknessObservations.has(skillName)) {
+      this.recurringWeaknessObservations.set(skillName, []);
+    }
+    const obs = this.recurringWeaknessObservations.get(skillName);
+    obs.push({ weakness, task, observed_at: new Date().toISOString() });
+
+    const matchingWeaknesses = obs.filter(
+      o =>
+        o.weakness.toLowerCase().includes(weakness.toLowerCase()) ||
+        weakness.toLowerCase().includes(o.weakness.toLowerCase()),
+    );
+
+    if (matchingWeaknesses.length >= 2) {
+      return this.generateProposal({
+        target_skill: skillName,
+        observed_weakness: weakness,
+        evidence: matchingWeaknesses.map(m => `Task '${m.task}' exposed weakness: ${m.weakness}`),
+        proposed_enhancement: `Enhance ${skillName} with explicit heuristics and checks for ${weakness}`,
+      });
+    }
+
+    return null;
+  }
+
+  /**
+   * Generates a formal Skill Improvement Proposal (SIP).
+   * Strictly enforces governance gating: Never silently self-modifies.
+   *
+   * @param {Object} params
+   * @returns {Object} Proposal record
+   */
+  generateProposal(params = {}) {
+    const id = `sip_${Date.now()}_${Math.random().toString(36).substring(2, 6)}`;
+    const proposal = {
+      id,
+      target_skill: params.target_skill || 'generic-skill',
+      observed_weakness: params.observed_weakness || 'Missing engineering consideration',
+      evidence: Array.isArray(params.evidence)
+        ? params.evidence
+        : [params.evidence || 'Multiple task regressions'],
+      proposed_enhancement:
+        params.proposed_enhancement ||
+        'Update SKILL.md with missing verification and consideration patterns',
+      status: PROPOSAL_STATUS.PENDING_REVIEW,
+      governance_gate: {
+        requires_human_approval: true,
+        self_modification_allowed: false,
+        warning: 'DO NOT SILENTLY SELF-MODIFY. Controlled governance review required.',
+      },
+      created_at: new Date().toISOString(),
+      reviewed_at: null,
+      review_notes: null,
+    };
+
+    this.proposals.set(id, proposal);
+    return proposal;
+  }
+
+  /**
+   * Simulates or executes Human / Governance review of a proposal.
+   *
+   * @param {string} proposalId
+   * @param {'APPROVED'|'REJECTED'} decision
+   * @param {string} rationale
+   * @returns {Object} Updated proposal
+   */
+  reviewProposal(proposalId, decision, rationale = '') {
+    const proposal = this.proposals.get(proposalId);
+    if (!proposal) {
+      throw new Error(`Proposal not found: ${proposalId}`);
+    }
+
+    if (![PROPOSAL_STATUS.APPROVED, PROPOSAL_STATUS.REJECTED].includes(decision)) {
+      throw new Error(`Invalid decision: ${decision}. Must be APPROVED or REJECTED.`);
+    }
+
+    proposal.status = decision;
+    proposal.reviewed_at = new Date().toISOString();
+    proposal.review_notes = rationale;
+
+    return proposal;
+  }
+
+  getProposals() {
+    return Array.from(this.proposals.values());
+  }
+}
+
+let _globalImprovementManager = null;
+function getSkillImprovementManager() {
+  if (!_globalImprovementManager) {
+    _globalImprovementManager = new SkillImprovementManager();
+  }
+  return _globalImprovementManager;
+}
+
+// ============================================================================
+// 4. PHASE 5 METRICS TRACKER (SECTION 27)
+// ============================================================================
+
+class Phase5MetricsTracker {
+  constructor() {
+    this.counters = {
+      total_retrieval_queries: 0,
+      relevant_memories_retrieved: 0,
+      irrelevant_memories_retrieved: 0,
+      relevant_memories_available: 0,
+      useful_memories_applied: 0,
+      false_memories_activated: 0,
+      memories_verified: 0,
+      memories_contradicted: 0,
+      stale_memories_detected: 0,
+      repeated_failures_prevented: 0,
+      skill_routing_improvements: 0,
+      consideration_recall_improvements: 0,
+      verification_strategies_reused: 0,
+      false_transfers_prevented: 0,
+      poisoned_memories_blocked: 0,
+      cross_project_leakages_prevented: 0,
+    };
+  }
+
+  recordEvent(eventName, count = 1) {
+    if (this.counters[eventName] !== undefined) {
+      this.counters[eventName] += count;
+    }
+  }
+
+  getMetrics() {
+    const c = this.counters;
+    const totalRetrieved = c.relevant_memories_retrieved + c.irrelevant_memories_retrieved;
+    const precision = totalRetrieved > 0 ? c.relevant_memories_retrieved / totalRetrieved : 1.0;
+    const recall =
+      c.relevant_memories_available > 0
+        ? c.relevant_memories_retrieved / c.relevant_memories_available
+        : 1.0;
+    const usefulRate = totalRetrieved > 0 ? c.useful_memories_applied / totalRetrieved : 1.0;
+
+    return {
+      memory_retrieval_precision: Math.round(precision * 100) / 100,
+      memory_retrieval_recall: Math.round(recall * 100) / 100,
+      useful_memory_rate: Math.round(usefulRate * 100) / 100,
+      false_memory_activation: c.false_memories_activated,
+      memory_verification_rate: c.memories_verified,
+      memory_contradiction_rate: c.memories_contradicted,
+      stale_memory_detection: c.stale_memories_detected,
+      repeated_failure_prevention: c.repeated_failures_prevented,
+      skill_routing_improvement: c.skill_routing_improvements,
+      consideration_recall_improvement: c.consideration_recall_improvements,
+      verification_reuse: c.verification_strategies_reused,
+      false_transfer_rate: c.false_transfers_prevented,
+      memory_poisoning_detection: c.poisoned_memories_blocked,
+      cross_project_leakage: c.cross_project_leakages_prevented,
+      raw_counters: { ...this.counters },
+    };
+  }
+}
+
+let _globalMetricsTracker = null;
+function getPhase5MetricsTracker() {
+  if (!_globalMetricsTracker) {
+    _globalMetricsTracker = new Phase5MetricsTracker();
+  }
+  return _globalMetricsTracker;
+}
+
+// ============================================================================
+// EXPORTS
+// ============================================================================
+
+module.exports = {
+  SkillTelemetryRegistry,
+  getSkillTelemetryRegistry,
+  CapabilityGapTracker,
+  getCapabilityGapTracker,
+  PROPOSAL_STATUS,
+  SkillImprovementManager,
+  getSkillImprovementManager,
+  Phase5MetricsTracker,
+  getPhase5MetricsTracker,
+};

package/.agent/scripts/concept_extractor.js

@@ -0,0 +1,240 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const _taxonomyCache = new Map();
+
+function clearTaxonomyCache() {
+  _taxonomyCache.clear();
+}
+
+function loadTaxonomy(agentDir) {
+  const cacheKey = path.resolve(agentDir || process.cwd());
+  if (_taxonomyCache.has(cacheKey)) {
+    return _taxonomyCache.get(cacheKey);
+  }
+
+  const possiblePaths = [
+    path.join(agentDir, 'skill-intelligence', 'taxonomy', 'engineering-concepts.json'),
+    path.join(
+      agentDir,
+      '..',
+      '.agent',
+      'skill-intelligence',
+      'taxonomy',
+      'engineering-concepts.json',
+    ),
+    path.join(__dirname, '..', 'skill-intelligence', 'taxonomy', 'engineering-concepts.json'),
+    path.join(
+      __dirname,
+      '..',
+      '.agent',
+      'skill-intelligence',
+      'taxonomy',
+      'engineering-concepts.json',
+    ),
+  ];
+
+  for (const p of possiblePaths) {
+    if (fs.existsSync(p)) {
+      try {
+        const concepts = JSON.parse(fs.readFileSync(p, 'utf8')).concepts || [];
+        _taxonomyCache.set(cacheKey, concepts);
+        return concepts;
+      } catch (_) {}
+    }
+  }
+  return [];
+}
+
+const ACTION_VERBS = [
+  'build',
+  'create',
+  'implement',
+  'optimize',
+  'tune',
+  'refactor',
+  'secure',
+  'audit',
+  'deploy',
+  'migrate',
+  'test',
+  'sync',
+  'fix',
+];
+const HIGH_RISK_TRIGGERS = [
+  {
+    match: /(?:payment|charge|checkout|money|billing|wallet|order)/i,
+    risk: 'Financial Transactions & Double Charge Risk',
+  },
+  {
+    match: /(?:auth|jwt|password|credential|secret|token|rbac|permission)/i,
+    risk: 'Authentication & Privilege Escalation Risk',
+  },
+  {
+    match: /(?:shell|exec|command|sandbox|subprocess|eval)/i,
+    risk: 'Arbitrary Code Execution & Tool Injection Risk',
+  },
+  {
+    match: /(?:concurrency|concurrent|race condition|lock|thread|parallel)/i,
+    risk: 'Race Condition & State Corruption Risk',
+  },
+  {
+    match: /(?:file|upload|multipart|attachment)/i,
+    risk: 'Unrestricted File Upload & Storage Exhaustion Risk',
+  },
+  {
+    match: /(?:realtime|real-time|websocket|collaborative|socket)/i,
+    risk: 'State Desynchronization & Concurrency Conflict Risk',
+  },
+];
+
+const DOMAIN_TRIGGERS = [
+  {
+    match: /(?:ui|frontend|css|responsive|html|page|landing|component|a11y|accessibility)/i,
+    domain: 'frontend',
+  },
+  { match: /(?:backend|api|server|endpoint|route|service)/i, domain: 'backend' },
+  { match: /(?:database|sql|postgres|mysql|query|index|table)/i, domain: 'database' },
+  {
+    match: /(?:distributed|worker|queue|cluster|multi-region|consensus|socket|realtime)/i,
+    domain: 'distributed-systems',
+  },
+  { match: /(?:security|auth|jwt|password|token|permission|sandbox)/i, domain: 'security' },
+  { match: /(?:kubernetes|k8s|docker|deploy|ci\/cd|pipeline|devops)/i, domain: 'devops' },
+  { match: /(?:ai|llm|rag|vector|prompt|agent)/i, domain: 'ai' },
+  { match: /(?:data|etl|pipeline|stream|batch)/i, domain: 'data-engineering' },
+  { match: /(?:search|index|ranking|relevance)/i, domain: 'search' },
+];
+
+function extractConcepts(taskQuery, agentDir) {
+  const q = (taskQuery || '').toLowerCase();
+  const taxonomy = loadTaxonomy(agentDir || process.cwd());
+
+  const explicit = [];
+  const implicitMap = new Map();
+  const domains = new Set();
+  const riskSignals = [];
+
+  // Detect domain triggers
+  for (const dt of DOMAIN_TRIGGERS) {
+    if (dt.match.test(q)) {
+      domains.add(dt.domain);
+    }
+  }
+
+  // Detect action verbs
+  const actions = ACTION_VERBS.filter(v => new RegExp(`\\b${v}\\b`, 'i').test(q));
+
+  // Detect risk signals
+  for (const r of HIGH_RISK_TRIGGERS) {
+    if (r.match.test(q)) {
+      riskSignals.push(r.risk);
+    }
+  }
+
+  // 1. Explicit matching against taxonomy
+  for (const item of taxonomy) {
+    let isExplicit = false;
+    const targets = [
+      item.id,
+      item.id.replace(/-/g, ' '),
+      item.name.toLowerCase(),
+      ...(item.aliases || []).map(a => a.toLowerCase()),
+    ];
+
+    for (const target of targets) {
+      if (target.length < 3) continue; // Skip very short abbreviations unless explicitly bounded
+      const escaped = target.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+      const re = new RegExp(`\\b${escaped}\\b`, 'i');
+      if (re.test(q)) {
+        isExplicit = true;
+        break;
+      }
+    }
+
+    if (isExplicit) {
+      explicit.push(item);
+      domains.add(item.domain);
+    }
+  }
+
+  // 2. Implicit matching (domain inference, related concepts, failure modes)
+  for (const item of taxonomy) {
+    if (explicit.some(e => e.id === item.id)) continue;
+
+    let impliedReason = null;
+
+    if (item.failure_modes && item.failure_modes.some(fm => q.includes(fm.replace(/-/g, ' ')))) {
+      impliedReason = 'Matched failure mode';
+    }
+
+    if (!impliedReason) {
+      if (
+        /(?:payment|checkout|order|charge)/i.test(q) &&
+        ['idempotency', 'transaction-integrity', 'concurrency-control'].includes(item.id)
+      ) {
+        impliedReason = 'Required for financial mutation consistency';
+      } else if (
+        /(?:postgres|sql|database|query)/i.test(q) &&
+        ['query-optimization', 'transaction-integrity'].includes(item.id)
+      ) {
+        impliedReason = 'Required for database reliability and performance';
+      } else if (
+        /(?:upload|file|attachment)/i.test(q) &&
+        ['file-upload-and-storage', 'input-validation-and-sanitization'].includes(item.id)
+      ) {
+        impliedReason = 'Required for storage security and MIME integrity';
+      } else if (
+        /(?:agent|shell|tool|prompt)/i.test(q) &&
+        ['ai-agent-security-and-sandboxing', 'input-validation-and-sanitization'].includes(item.id)
+      ) {
+        impliedReason = 'Required for agent runtime safety and boundary defense';
+      } else if (
+        /(?:collaborative|realtime|real-time|editor)/i.test(q) &&
+        ['realtime-collaboration-and-sync', 'concurrency-control'].includes(item.id)
+      ) {
+        impliedReason = 'Required for conflict-free state synchronization';
+      }
+    }
+
+    if (!impliedReason) {
+      for (const exp of explicit) {
+        if (exp.related && exp.related.includes(item.id)) {
+          impliedReason = `Related to explicit concept '${exp.name}'`;
+          break;
+        }
+      }
+    }
+
+    if (impliedReason) {
+      implicitMap.set(item.id, { ...item, implied_reason: impliedReason });
+      domains.add(item.domain);
+    }
+  }
+
+  const implicit = Array.from(implicitMap.values());
+  const all = [...explicit, ...implicit];
+
+  return {
+    query: taskQuery,
+    explicit_concepts: explicit.map(c => ({ id: c.id, name: c.name, domain: c.domain })),
+    implicit_concepts: implicit.map(c => ({
+      id: c.id,
+      name: c.name,
+      domain: c.domain,
+      reason: c.implied_reason,
+    })),
+    all_concepts: all,
+    domains: Array.from(domains),
+    actions,
+    risk_signals: riskSignals,
+  };
+}
+
+module.exports = {
+  extractConcepts,
+  loadTaxonomy,
+  clearTaxonomyCache,
+};