scene-capability-engine 3.6.65 → 3.6.67

package/lib/semantic/progress-ledger.js

@@ -0,0 +1,433 @@
+const { getSceStateStore } = require('../state/sce-state-store');
+const { buildSemanticArchiveSummary } = require('./archive-report');
+const { buildSupremeIntentGovernanceReport } = require('./supreme-intent-report');
+const { buildSemanticBackflowReport } = require('./backflow-report');
+const { buildSemanticArchiveReport } = require('./archive-report');
+const { buildSemanticSharedSourceProjection } = require('../project/semantic-shared-source-projection');
+const { buildSupervisionActionQueue } = require('../project/supervision-action-model');
+
+function normalizeString(value) {
+  if (typeof value !== 'string') {
+    return '';
+  }
+  return value.trim();
+}
+
+function normalizeInteger(value, fallback = 0) {
+  const parsed = Number.parseInt(`${value}`, 10);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    return fallback;
+  }
+  return parsed;
+}
+
+function percent(part, total) {
+  if (!Number.isFinite(part) || !Number.isFinite(total) || total <= 0) {
+    return 0;
+  }
+  return Number(((part / total) * 100).toFixed(2));
+}
+
+function readObservationMetadata(observation = {}) {
+  if (observation.normalized_payload && typeof observation.normalized_payload === 'object'
+      && observation.normalized_payload.metadata
+      && typeof observation.normalized_payload.metadata === 'object') {
+    return observation.normalized_payload.metadata;
+  }
+  return {};
+}
+
+function readLessonPublishState(lesson = {}) {
+  if (lesson.metadata && typeof lesson.metadata === 'object'
+      && lesson.metadata.publish
+      && typeof lesson.metadata.publish === 'object') {
+    return normalizeString(lesson.metadata.publish.publish_state) || 'none';
+  }
+  return 'none';
+}
+
+function buildGovernanceOperatorItems(report = {}) {
+  const hotspots = Array.isArray(report.top_scope_hotspots) ? report.top_scope_hotspots : [];
+  return hotspots
+    .filter((item) => Number(item && item.governed_count || 0) > 0)
+    .slice(0, 3)
+    .map((item) => ({
+      id: `semantic-progress:governance:${normalizeString(item.project_id) || 'project'}:${normalizeString(item.channel_id) || 'channel'}:${normalizeString(item.spec_id) || 'spec'}`,
+      kind: 'governance',
+      state: Number(item.refuse || 0) > 0 ? 'refuse-hotspot' : 'rewrite-hotspot',
+      channelId: normalizeString(item.channel_id) || null,
+      sceneId: normalizeString(item.scene_id) || null,
+      specId: normalizeString(item.spec_id) || null,
+      governedCount: Number(item.governed_count || 0),
+      updatedAt: normalizeString(item.latest_at) || null,
+      summary: `Governance hotspot on ${normalizeString(item.channel_id) || 'project'} / ${normalizeString(item.spec_id) || normalizeString(item.scene_id) || 'unbound'} with ${Number(item.governed_count || 0)} governed assessments.`
+    }));
+}
+
+function buildBackflowOperatorItems(report = {}) {
+  const items = [];
+  const localBlockedItems = Array.isArray(report?.local?.recent_blocked_items)
+    ? report.local.recent_blocked_items.slice(0, 3)
+    : [];
+  for (const item of localBlockedItems) {
+    items.push({
+      id: `semantic-progress:backflow:local:${normalizeString(item.lesson_id) || normalizeString(item.capability_id) || normalizeString(item.spec_id) || 'unknown'}`,
+      kind: 'backflow',
+      state: 'local-publish-blocked',
+      channelId: normalizeString(item.channel_id) || null,
+      sceneId: normalizeString(item.scene_id) || null,
+      specId: normalizeString(item.spec_id) || null,
+      blockedReason: normalizeString(item.blocked_reason) || 'unknown',
+      blockedCount: 1,
+      updatedAt: normalizeString(item.blocked_at) || null,
+      summary: `Local semantic publication is blocked for ${normalizeString(item.spec_id) || normalizeString(item.scene_id) || 'project scope'} (${normalizeString(item.blocked_reason) || 'unknown'}).`
+    });
+  }
+
+  const mergeReceipts = Array.isArray(report?.central_merge?.receipts) ? report.central_merge.receipts : [];
+  for (const receipt of mergeReceipts.filter((entry) => Number(entry?.totals?.blocked || 0) > 0).slice(0, 3)) {
+    items.push({
+      id: `semantic-progress:backflow:merge:${normalizeString(receipt.spec_id) || 'unknown'}:${normalizeString(receipt.generated_at) || 'latest'}`,
+      kind: 'backflow',
+      state: 'central-merge-blocked',
+      specId: normalizeString(receipt.spec_id) || null,
+      blockedCount: Number(receipt?.totals?.blocked || 0),
+      blockedReasons: (Array.isArray(receipt.blocked) ? receipt.blocked : [])
+        .map((entry) => normalizeString(entry && entry.reason))
+        .filter(Boolean),
+      updatedAt: normalizeString(receipt.generated_at) || null,
+      summary: `Central merge is blocked for ${normalizeString(receipt.spec_id) || 'unknown spec'} with ${Number(receipt?.totals?.blocked || 0)} blocked entr${Number(receipt?.totals?.blocked || 0) === 1 ? 'y' : 'ies'}.`
+    });
+  }
+
+  const releaseReceipts = Array.isArray(report?.central_release?.receipts) ? report.central_release.receipts : [];
+  for (const receipt of releaseReceipts.filter((entry) => Number(entry?.totals?.blocked || 0) > 0).slice(0, 3)) {
+    items.push({
+      id: `semantic-progress:backflow:release:${normalizeString(receipt.spec_id) || 'unknown'}:${normalizeString(receipt.generated_at) || 'latest'}`,
+      kind: 'backflow',
+      state: 'central-release-blocked',
+      specId: normalizeString(receipt.spec_id) || null,
+      blockedCount: Number(receipt?.totals?.blocked || 0),
+      blockedReasons: (Array.isArray(receipt.blocked) ? receipt.blocked : [])
+        .map((entry) => normalizeString(entry && entry.reason))
+        .filter(Boolean),
+      updatedAt: normalizeString(receipt.generated_at) || null,
+      summary: `Approved-central release is blocked for ${normalizeString(receipt.spec_id) || 'unknown spec'} with ${Number(receipt?.totals?.blocked || 0)} blocked entr${Number(receipt?.totals?.blocked || 0) === 1 ? 'y' : 'ies'}.`
+    });
+  }
+
+  return items;
+}
+
+function buildArchiveOperatorItems(report = {}) {
+  const items = [];
+  const libraries = Array.isArray(report.by_library) ? report.by_library : [];
+  for (const item of libraries.filter((entry) => (
+    Number(entry.lesson_count || 0) > 0
+    || Number(entry.ledger_count || 0) > 0
+  )).slice(0, 3)) {
+    items.push({
+      id: `semantic-progress:archive:${normalizeString(item.target_library) || 'unknown'}`,
+      kind: 'archive',
+      state: Number(item.published_shared_count || 0) > 0 ? 'shared-accumulating' : 'local-accumulating',
+      targetLibrary: normalizeString(item.target_library) || null,
+      publishPendingCount: Number(item.publish_pending_count || 0),
+      publishBlockedCount: Number(item.publish_blocked_count || 0),
+      sessionId: null,
+      specId: null,
+      updatedAt: null,
+      summary: `${normalizeString(item.target_library) || 'unknown'} is accumulating ${Number(item.lesson_count || 0)} lesson(s) and ${Number(item.ledger_count || 0)} ledger row(s).`
+    });
+  }
+
+  if (Number(report?.totals?.missing_archive_count || 0) > 0) {
+    items.push({
+      id: 'semantic-progress:archive:missing-routing',
+      kind: 'archive',
+      state: 'archive-routing-missing',
+      missingArchiveCount: Number(report.totals.missing_archive_count || 0),
+      updatedAt: normalizeString(report.generated_at) || null,
+      summary: `${Number(report.totals.missing_archive_count || 0)} archive item(s) are missing normalized routing metadata.`
+    });
+  }
+
+  return items;
+}
+
+function buildSharedSourceOperatorItems(projection = {}, specId = '') {
+  const normalizedSpecId = normalizeString(specId);
+  const items = Array.isArray(projection.items) ? projection.items : [];
+  return items
+    .filter((item) => item.approved && !item.connected)
+    .filter((item) => !normalizedSpecId || normalizeString(item.spec_id) === normalizedSpecId)
+    .map((item) => ({
+      id: `semantic-progress:shared-source:${normalizeString(item.source_name) || 'source'}:${normalizeString(item.descriptor_file) || 'descriptor'}`,
+      kind: 'info',
+      state: 'pending-connect',
+      specId: normalizeString(item.spec_id) || null,
+      descriptorFile: normalizeString(item.descriptor_file) || null,
+      sourceName: normalizeString(item.source_name) || null,
+      suggestedConnectCommand: normalizeString(item.suggested_connect_command) || null,
+      updatedAt: normalizeString(item.updated_at) || null,
+      summary: `Approved semantic shared source ${normalizeString(item.source_name) || 'source'} is available but not connected.`
+    }));
+}
+
+async function buildSemanticProgressReport(options = {}, dependencies = {}) {
+  const projectPath = dependencies.projectPath || process.cwd();
+  const store = dependencies.stateStore || getSceStateStore(projectPath, {
+    fileSystem: dependencies.fileSystem,
+    env: dependencies.env,
+    sqliteModule: dependencies.sqliteModule,
+    noCache: dependencies.noCache === true
+  });
+
+  const projectId = normalizeString(options.project_id || options.projectId);
+  const specId = normalizeString(options.spec_id || options.specId);
+  const sessionId = normalizeString(options.session_id || options.sessionId);
+  const scanLimit = normalizeInteger(options.limit, 500);
+
+  let observations = await store.listSemanticObservations({
+    project_id: projectId,
+    session_id: sessionId,
+    limit: scanLimit
+  });
+  observations = Array.isArray(observations) ? observations : [];
+  if (specId) {
+    observations = observations.filter((item) => normalizeString(item.spec_id) === specId);
+  }
+
+  let lessons = await store.listSemanticLessonCandidates({
+    project_id: projectId,
+    limit: scanLimit
+  });
+  lessons = Array.isArray(lessons) ? lessons : [];
+  if (specId) {
+    lessons = lessons.filter((item) => normalizeString(item.spec_id) === specId);
+  }
+
+  let evaluationRuns = await store.listSemanticEvaluationRuns({
+    project_id: projectId,
+    session_id: sessionId,
+    spec_id: specId,
+    limit: scanLimit
+  });
+  evaluationRuns = Array.isArray(evaluationRuns) ? evaluationRuns : [];
+
+  let capabilityLedgers = await store.listSemanticCapabilityLedgers({
+    project_id: projectId,
+    spec_id: specId,
+    limit: scanLimit
+  });
+  capabilityLedgers = Array.isArray(capabilityLedgers) ? capabilityLedgers : [];
+  if (sessionId) {
+    capabilityLedgers = capabilityLedgers.filter((item) => normalizeString(item.session_id) === sessionId);
+  }
+
+  let supremeIntentAssessments = await store.listSupremeIntentAssessments({
+    project_id: projectId,
+    session_id: sessionId,
+    spec_id: specId,
+    limit: scanLimit
+  });
+  supremeIntentAssessments = Array.isArray(supremeIntentAssessments) ? supremeIntentAssessments : [];
+
+  const nativeObservations = observations.filter((item) => normalizeString(item.source_runtime) === 'sce-native');
+  const nativePromptCount = nativeObservations.filter((item) => normalizeString(item.event_kind) === 'prompt').length;
+  const nativeReplyItems = nativeObservations.filter((item) => normalizeString(item.event_kind) === 'reply');
+  const activatedReplyItems = nativeReplyItems.filter((item) => {
+    const metadata = readObservationMetadata(item);
+    return Array.isArray(metadata.activated_lesson_ids) && metadata.activated_lesson_ids.length > 0;
+  });
+
+  const activatedCapabilityIds = new Set();
+  for (const item of activatedReplyItems) {
+    const metadata = readObservationMetadata(item);
+    const ids = Array.isArray(metadata.activated_capability_ids)
+      ? metadata.activated_capability_ids
+      : (Array.isArray(metadata.activated_lesson_ids) ? metadata.activated_lesson_ids : []);
+    for (const id of ids) {
+      const normalizedId = normalizeString(id);
+      if (normalizedId) {
+        activatedCapabilityIds.add(normalizedId);
+      }
+    }
+  }
+
+  const promotedLessons = lessons.filter((item) => ['promoted', 'promoted-local', 'published-shared', 'publish_pending'].includes(normalizeString(item.status)));
+  const deprecatedLessons = lessons.filter((item) => normalizeString(item.status) === 'deprecated');
+  const candidateLessons = lessons.filter((item) => normalizeString(item.status) === 'candidate');
+  const publishPendingLessons = lessons.filter((item) => readLessonPublishState(item) === 'pending' || normalizeString(item.status) === 'publish_pending');
+  const publishedSharedLessons = lessons.filter((item) => readLessonPublishState(item) === 'published-shared' || normalizeString(item.status) === 'published-shared');
+  const publishBlockedLessons = lessons.filter((item) => readLessonPublishState(item) === 'blocked' || normalizeString(item.status) === 'publish_blocked');
+  const passedEvaluations = evaluationRuns.filter((item) => normalizeString(item.verdict) === 'passed');
+  const failedEvaluations = evaluationRuns.filter((item) => normalizeString(item.verdict) === 'failed');
+  const regressionEvaluations = evaluationRuns.filter((item) => item.regression_detected === true);
+  const activeCapabilities = capabilityLedgers.filter((item) => normalizeString(item.capability_status) === 'active');
+  const quarantinedCapabilities = capabilityLedgers.filter((item) => normalizeString(item.capability_status) === 'quarantined');
+  const deprecatedCapabilities = capabilityLedgers.filter((item) => normalizeString(item.capability_status) === 'deprecated');
+  const blockedCapabilities = capabilityLedgers.filter((item) => normalizeString(item.publish_state) === 'blocked');
+  const governancePenalizedCapabilities = capabilityLedgers.filter((item) => Number(item.metrics && item.metrics.governance_penalty_score) > 0);
+  const governanceHighRiskCapabilities = capabilityLedgers.filter((item) => Number(item.metrics && item.metrics.governance_penalty_score) >= 20);
+  const supremeAllow = supremeIntentAssessments.filter((item) => normalizeString(item.action) === 'allow');
+  const supremeClarify = supremeIntentAssessments.filter((item) => normalizeString(item.action) === 'clarify');
+  const supremeRewrite = supremeIntentAssessments.filter((item) => normalizeString(item.action) === 'rewrite');
+  const supremeNarrow = supremeIntentAssessments.filter((item) => normalizeString(item.action) === 'narrow');
+  const supremeRefuse = supremeIntentAssessments.filter((item) => normalizeString(item.action) === 'refuse');
+  const planningBlocked = supremeIntentAssessments.filter((item) => item.planning_allowed !== true);
+  const executionBlocked = supremeIntentAssessments.filter((item) => item.execution_allowed !== true);
+
+  const supremeBySource = {};
+  for (const item of supremeIntentAssessments) {
+    const sourceKind = normalizeString(item.source_kind) || 'unknown';
+    supremeBySource[sourceKind] = (supremeBySource[sourceKind] || 0) + 1;
+  }
+
+  const runtimeBreakdown = {};
+  for (const item of observations) {
+    const runtime = normalizeString(item.source_runtime) || 'unknown';
+    runtimeBreakdown[runtime] = (runtimeBreakdown[runtime] || 0) + 1;
+  }
+  const archiveSummary = buildSemanticArchiveSummary({
+    lessons,
+    capabilityLedgers,
+    recent_limit: 10
+  });
+  const governanceReport = await buildSupremeIntentGovernanceReport({
+    project_id: projectId,
+    spec_id: specId,
+    session_id: sessionId,
+    limit: scanLimit,
+    hotspot_limit: 3,
+    recent_limit: 5
+  }, dependencies);
+  const backflowReport = await buildSemanticBackflowReport({
+    project_id: projectId,
+    spec_id: specId,
+    limit: scanLimit,
+    recent_limit: 5
+  }, dependencies);
+  const archiveReport = await buildSemanticArchiveReport({
+    project_id: projectId,
+    spec_id: specId,
+    session_id: sessionId,
+    limit: scanLimit,
+    recent_limit: 5
+  }, dependencies);
+  const semanticSharedSources = await buildSemanticSharedSourceProjection(projectPath, {
+    fileSystem: dependencies.fileSystem
+  });
+  const operatorItems = [
+    ...buildGovernanceOperatorItems(governanceReport),
+    ...buildBackflowOperatorItems(backflowReport),
+    ...buildArchiveOperatorItems(archiveReport),
+    ...buildSharedSourceOperatorItems(semanticSharedSources, specId)
+  ];
+  const operatorProjection = buildSupervisionActionQueue(operatorItems, {
+    projectId
+  });
+
+  const funnel = {
+    observations: observations.length,
+    lesson_candidates: lessons.length,
+    evaluation_runs: evaluationRuns.length,
+    promoted_capabilities: promotedLessons.length,
+    native_activated_replies: activatedReplyItems.length,
+    supreme_intent_assessments: supremeIntentAssessments.length
+  };
+
+  return {
+    mode: 'semantic-progress',
+    success: true,
+    generated_at: new Date().toISOString(),
+    filters: {
+      project_id: projectId || null,
+      spec_id: specId || null,
+      session_id: sessionId || null,
+      limit: scanLimit
+    },
+    funnel,
+    observations: {
+      total: observations.length,
+      native_total: nativeObservations.length,
+      native_prompt_count: nativePromptCount,
+      native_reply_count: nativeReplyItems.length,
+      runtime_breakdown: runtimeBreakdown
+    },
+    lesson_candidates: {
+      total: lessons.length,
+      candidate: candidateLessons.length,
+      promoted: promotedLessons.length,
+      deprecated: deprecatedLessons.length,
+      publish_pending: publishPendingLessons.length,
+      published_shared: publishedSharedLessons.length,
+      publish_blocked: publishBlockedLessons.length
+    },
+    evaluation: {
+      total: evaluationRuns.length,
+      passed: passedEvaluations.length,
+      failed: failedEvaluations.length,
+      regressions: regressionEvaluations.length,
+      pass_rate_percent: percent(passedEvaluations.length, evaluationRuns.length),
+      regression_rate_percent: percent(regressionEvaluations.length, evaluationRuns.length)
+    },
+    capability_ledger: {
+      total: capabilityLedgers.length,
+      active: activeCapabilities.length,
+      quarantined: quarantinedCapabilities.length,
+      deprecated: deprecatedCapabilities.length,
+      publish_blocked: blockedCapabilities.length,
+      governance_penalized: governancePenalizedCapabilities.length,
+      governance_high_risk: governanceHighRiskCapabilities.length
+    },
+    supreme_intent: {
+      total: supremeIntentAssessments.length,
+      allow: supremeAllow.length,
+      clarify: supremeClarify.length,
+      rewrite: supremeRewrite.length,
+      narrow: supremeNarrow.length,
+      refuse: supremeRefuse.length,
+      planning_blocked: planningBlocked.length,
+      execution_blocked: executionBlocked.length,
+      by_source: supremeBySource,
+      governed_action_rate_percent: percent(
+        supremeClarify.length + supremeRewrite.length + supremeNarrow.length + supremeRefuse.length,
+        supremeIntentAssessments.length
+      )
+    },
+    native_backflow: {
+      activated_reply_count: activatedReplyItems.length,
+      activation_hit_rate_percent: percent(activatedReplyItems.length, nativeReplyItems.length),
+      activated_capability_count: activatedCapabilityIds.size,
+      activated_capability_ids: [...activatedCapabilityIds].sort()
+    },
+    archive: archiveSummary,
+    shared_sources: semanticSharedSources.summary,
+    operator_items: operatorProjection.items,
+    action_queue: operatorProjection.actionQueue,
+    operator_control_plane: {
+      ...operatorProjection.summary,
+      governanceHotspotCount: buildGovernanceOperatorItems(governanceReport).length,
+      backflowItemCount: buildBackflowOperatorItems(backflowReport).length,
+      archiveItemCount: buildArchiveOperatorItems(archiveReport).length,
+      pendingConnectCount: Number(semanticSharedSources.summary.pendingApprovedDescriptors || 0)
+    },
+    progress_signals: {
+      distillation_ready: lessons.length > 0,
+      evaluation_ready: evaluationRuns.length > 0,
+      promotion_ready: passedEvaluations.some((item) => item.governance && item.governance.promotion_gate_passed === true),
+      native_runtime_learning_active: nativeObservations.length > 0,
+      supreme_governance_active: supremeIntentAssessments.length > 0,
+      standalone_substitution_progress_percent: Number((
+        (percent(promotedLessons.length > 0 ? 1 : 0, 1) * 0.35)
+        + (percent(evaluationRuns.length > 0 ? 1 : 0, 1) * 0.25)
+        + (percent(nativeObservations.length > 0 ? 1 : 0, 1) * 0.2)
+        + (percent(activatedReplyItems.length > 0 ? 1 : 0, 1) * 0.2)
+      ).toFixed(2))
+    }
+  };
+}
+
+module.exports = {
+  buildSemanticProgressReport
+};