scene-capability-engine 3.6.65 → 3.6.67

package/lib/semantic/shared-sync-merge.js

@@ -0,0 +1,547 @@
+const path = require('path');
+const http = require('http');
+const https = require('https');
+const fs = require('fs-extra');
+const {
+  SEMANTIC_SHARED_SYNC_BUNDLE_API_VERSION,
+  SEMANTIC_SHARED_SYNC_INDEX_API_VERSION,
+  SEMANTIC_SHARED_SYNC_SHARD_API_VERSION,
+  resolveDefaultSemanticSharedSyncBundleOutFile,
+  resolveDefaultSemanticSharedSyncIndexOutFile,
+  resolveDefaultSemanticSharedSyncReceiptOutFile
+} = require('./shared-sync-export');
+
+const SEMANTIC_SHARED_SYNC_MERGE_RECEIPT_API_VERSION = 'sce.semantic.shared-sync.merge-receipt/v0.1';
+
+function normalizeString(value) {
+  if (typeof value !== 'string') {
+    return '';
+  }
+  return value.trim();
+}
+
+function normalizeInteger(value, fallback = 0) {
+  const parsed = Number.parseInt(`${value}`, 10);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    return fallback;
+  }
+  return fallback > 0 ? parsed : parsed;
+}
+
+function normalizeStringList(...values) {
+  const results = [];
+  for (const value of values) {
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        const normalized = normalizeString(item);
+        if (normalized) {
+          results.push(normalized);
+        }
+      }
+      continue;
+    }
+    if (typeof value === 'string') {
+      value.split(',')
+        .map((item) => normalizeString(item))
+        .filter(Boolean)
+        .forEach((item) => results.push(item));
+    }
+  }
+  return Array.from(new Set(results));
+}
+
+function isObject(value) {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}
+
+function isHttpSource(source = '') {
+  return /^https?:\/\//i.test(normalizeString(source));
+}
+
+function toProjectRelative(projectPath, targetPath) {
+  return path.relative(projectPath, targetPath).replace(/\\/g, '/');
+}
+
+function resolveDefaultSemanticSharedMergeBaseDir(specId = '') {
+  const normalizedSpecId = normalizeString(specId);
+  if (!normalizedSpecId) {
+    return '';
+  }
+  return `.sce/specs/${normalizedSpecId}/registry/semantic-shared`;
+}
+
+function resolveDefaultSemanticSharedMergeBundleOutFile(specId = '') {
+  const baseDir = resolveDefaultSemanticSharedMergeBaseDir(specId);
+  return baseDir ? `${baseDir}/latest.bundle.json` : '';
+}
+
+function resolveDefaultSemanticSharedMergeIndexOutFile(specId = '') {
+  const baseDir = resolveDefaultSemanticSharedMergeBaseDir(specId);
+  return baseDir ? `${baseDir}/latest.index.json` : '';
+}
+
+function resolveDefaultSemanticSharedMergeReceiptOutFile(specId = '') {
+  const baseDir = resolveDefaultSemanticSharedMergeBaseDir(specId);
+  return baseDir ? `${baseDir}/latest.receipt.json` : '';
+}
+
+function resolveDefaultSemanticSharedMergeShardOutFile(specId = '', targetLibrary = '') {
+  const baseDir = resolveDefaultSemanticSharedMergeBaseDir(specId);
+  const normalizedLibrary = normalizeString(targetLibrary);
+  return baseDir && normalizedLibrary
+    ? `${baseDir}/shards/${normalizedLibrary}.json`
+    : '';
+}
+
+function fetchJsonFromHttp(source, timeoutMs = 15000) {
+  const normalized = normalizeString(source);
+  if (!normalized) {
+    return Promise.reject(new Error('shared merge source is required'));
+  }
+  const client = normalized.startsWith('https://') ? https : http;
+  return new Promise((resolve, reject) => {
+    const request = client.get(normalized, {
+      timeout: timeoutMs,
+      headers: {
+        Accept: 'application/json'
+      }
+    }, (response) => {
+      const chunks = [];
+      response.on('data', (chunk) => chunks.push(chunk));
+      response.on('end', () => {
+        const body = Buffer.concat(chunks).toString('utf8');
+        if (response.statusCode < 200 || response.statusCode >= 300) {
+          reject(new Error(`shared merge source responded ${response.statusCode}`));
+          return;
+        }
+        try {
+          resolve(JSON.parse(body));
+        } catch (error) {
+          reject(new Error(`shared merge source returned invalid JSON: ${error.message}`));
+        }
+      });
+    });
+    request.on('timeout', () => {
+      request.destroy(new Error('shared merge source request timed out'));
+    });
+    request.on('error', reject);
+  });
+}
+
+async function loadJsonSource(projectPath, source, fileSystem = fs) {
+  const normalized = normalizeString(source);
+  if (!normalized) {
+    throw new Error('shared merge source is required');
+  }
+  if (isHttpSource(normalized)) {
+    return fetchJsonFromHttp(normalized);
+  }
+  const absolutePath = path.isAbsolute(normalized)
+    ? normalized
+    : path.resolve(projectPath, normalized);
+  if (!await fileSystem.pathExists(absolutePath)) {
+    throw new Error(`shared merge source file not found: ${source}`);
+  }
+  return fileSystem.readJson(absolutePath);
+}
+
+function inferSourceRootFromSource(source = '') {
+  const normalized = normalizeString(source);
+  if (!normalized) {
+    return '';
+  }
+  if (isHttpSource(normalized)) {
+    const sceMarkerIndex = normalized.indexOf('/.sce/');
+    if (sceMarkerIndex > 0) {
+      return normalized.slice(0, sceMarkerIndex);
+    }
+    return normalizeString(normalized.replace(/\/[^/]*$/, ''));
+  }
+  const normalizedPath = path.resolve(normalized);
+  const marker = `${path.sep}.sce${path.sep}`;
+  const markerIndex = normalizedPath.indexOf(marker);
+  if (markerIndex > -1) {
+    return normalizedPath.slice(0, markerIndex);
+  }
+  return path.dirname(normalizedPath);
+}
+
+function resolveShardSource(sourceRoot = '', shardFile = '', bundleSource = '') {
+  const normalizedShardFile = normalizeString(shardFile);
+  if (!normalizedShardFile) {
+    return '';
+  }
+  if (isHttpSource(normalizedShardFile) || path.isAbsolute(normalizedShardFile)) {
+    return normalizedShardFile;
+  }
+  const normalizedRoot = normalizeString(sourceRoot) || inferSourceRootFromSource(bundleSource);
+  if (isHttpSource(normalizedRoot)) {
+    return `${normalizedRoot.replace(/\/+$/, '')}/${normalizedShardFile.replace(/^\/+/, '')}`;
+  }
+  if (normalizedRoot) {
+    return path.resolve(normalizedRoot, normalizedShardFile);
+  }
+  if (isHttpSource(bundleSource)) {
+    return new URL(normalizedShardFile, bundleSource).toString();
+  }
+  return path.resolve(path.dirname(bundleSource), normalizedShardFile);
+}
+
+async function readSemanticSharedSourceConfig(projectPath, configPath = '', fileSystem = fs) {
+  const normalizedConfigPath = normalizeString(configPath) || '.sce/config/semantic-shared-sources.json';
+  const absoluteConfigPath = path.isAbsolute(normalizedConfigPath)
+    ? normalizedConfigPath
+    : path.resolve(projectPath, normalizedConfigPath);
+  if (!await fileSystem.pathExists(absoluteConfigPath)) {
+    return {
+      enabled: true,
+      sources: []
+    };
+  }
+  const payload = await fileSystem.readJson(absoluteConfigPath);
+  if (!isObject(payload)) {
+    throw new Error(`semantic shared source config must be a JSON object: ${absoluteConfigPath}`);
+  }
+  return payload;
+}
+
+function resolveConfiguredSources(config = {}) {
+  const sources = Array.isArray(config.sources) ? config.sources : [];
+  return sources
+    .filter((item) => isObject(item) && item.enabled !== false)
+    .map((item) => ({
+      name: normalizeString(item.name) || 'semantic-shared',
+      bundle: normalizeString(item.bundle || item.source || item.url || item.file),
+      root: normalizeString(item.root || item.base || item.base_url || item.base_path)
+    }))
+    .filter((item) => item.bundle);
+}
+
+function comparePublishedAt(left = {}, right = {}) {
+  const leftTime = Date.parse(left.published_at || '') || 0;
+  const rightTime = Date.parse(right.published_at || '') || 0;
+  return leftTime - rightTime;
+}
+
+function assessCentralMergeEntryGate(entry = {}) {
+  const publication = isObject(entry.publication) ? entry.publication : {};
+  const publishState = normalizeString(publication.publish_state);
+  const blockedReason = normalizeString(publication.blocked_reason);
+  const blockedGate = isObject(publication.blocked_gate) ? publication.blocked_gate : null;
+  const reasons = [];
+
+  if (publishState && publishState !== 'published-shared') {
+    reasons.push({
+      reason: 'entry-not-shared-ready',
+      publish_state: publishState
+    });
+  }
+  if (blockedReason) {
+    reasons.push({
+      reason: 'entry-has-blocked-reason',
+      blocked_reason: blockedReason
+    });
+  }
+  if (blockedGate) {
+    reasons.push({
+      reason: 'entry-has-blocked-gate',
+      blocked_gate: blockedGate
+    });
+  }
+
+  return {
+    allowed: reasons.length === 0,
+    reasons
+  };
+}
+
+async function mergeSemanticSharedBundles(options = {}, dependencies = {}) {
+  const projectPath = dependencies.projectPath || process.cwd();
+  const fileSystem = dependencies.fileSystem || fs;
+  const config = await readSemanticSharedSourceConfig(projectPath, options.config, fileSystem);
+  const configuredSources = resolveConfiguredSources(config);
+  const explicitSources = normalizeStringList(options.source, options.sources);
+  const sourceRefs = explicitSources.length > 0
+    ? explicitSources.map((source, index) => ({
+      name: `source-${index + 1}`,
+      bundle: source,
+      root: ''
+    }))
+    : configuredSources;
+
+  if (sourceRefs.length === 0) {
+    throw new Error('at least one shared merge source is required (use --source/--sources or configure semantic-shared-sources.json)');
+  }
+
+  const specId = normalizeString(options.spec);
+  const limit = normalizeInteger(options.limit, 0);
+  const generatedAt = new Date().toISOString();
+  const candidateEntriesByKey = new Map();
+  const blocked = [];
+  const superseded = [];
+  let candidateEntries = 0;
+
+  const sourceReports = [];
+  for (const sourceRef of sourceRefs) {
+    const bundleSource = normalizeString(sourceRef.bundle);
+    const sourceRoot = normalizeString(sourceRef.root) || inferSourceRootFromSource(bundleSource);
+    const sourceReport = {
+      source: bundleSource,
+      source_name: normalizeString(sourceRef.name) || null,
+      bundle_ok: false,
+      shards_ok: 0,
+      entries_seen: 0
+    };
+
+    let bundlePayload;
+    try {
+      bundlePayload = await loadJsonSource(projectPath, bundleSource, fileSystem);
+    } catch (error) {
+      blocked.push({
+        source: bundleSource,
+        reason: 'failed-to-load-bundle',
+        detail: error.message
+      });
+      sourceReports.push(sourceReport);
+      continue;
+    }
+    if (!isObject(bundlePayload) || normalizeString(bundlePayload.api_version) !== SEMANTIC_SHARED_SYNC_BUNDLE_API_VERSION) {
+      blocked.push({
+        source: bundleSource,
+        reason: 'invalid-bundle-payload'
+      });
+      sourceReports.push(sourceReport);
+      continue;
+    }
+    sourceReport.bundle_ok = true;
+
+    const libraries = Array.isArray(bundlePayload.libraries) ? bundlePayload.libraries : [];
+    for (const library of libraries) {
+      const targetLibrary = normalizeString(library && library.target_library);
+      const shardSource = resolveShardSource(sourceRoot, normalizeString(library && library.shard_file), bundleSource);
+      if (!targetLibrary || !shardSource) {
+        blocked.push({
+          source: bundleSource,
+          target_library: targetLibrary || null,
+          reason: 'missing-target-library-or-shard-source'
+        });
+        continue;
+      }
+      let shardPayload;
+      try {
+        shardPayload = await loadJsonSource(projectPath, shardSource, fileSystem);
+      } catch (error) {
+        blocked.push({
+          source: bundleSource,
+          target_library: targetLibrary,
+          reason: 'failed-to-load-shard',
+          detail: error.message
+        });
+        continue;
+      }
+      if (!isObject(shardPayload) || normalizeString(shardPayload.api_version) !== SEMANTIC_SHARED_SYNC_SHARD_API_VERSION) {
+        blocked.push({
+          source: bundleSource,
+          target_library: targetLibrary,
+          reason: 'invalid-shard-payload'
+        });
+        continue;
+      }
+      sourceReport.shards_ok += 1;
+      const entries = Array.isArray(shardPayload.entries) ? shardPayload.entries : [];
+      for (const entry of entries) {
+        const capabilityId = normalizeString(entry && entry.capability_id);
+        const entryTargetLibrary = normalizeString(entry && entry.target_library) || targetLibrary;
+        if (!capabilityId || !entryTargetLibrary) {
+          blocked.push({
+            source: bundleSource,
+            target_library: entryTargetLibrary || null,
+            reason: 'missing-capability-or-library'
+          });
+          continue;
+        }
+        const gate = assessCentralMergeEntryGate(entry);
+        if (!gate.allowed) {
+          blocked.push({
+            source: bundleSource,
+            capability_id: capabilityId,
+            lesson_id: normalizeString(entry && entry.lesson_id) || null,
+            target_library: entryTargetLibrary,
+            reason: 'entry-failed-central-merge-gate',
+            gate
+          });
+          continue;
+        }
+        candidateEntries += 1;
+        sourceReport.entries_seen += 1;
+        const key = `${entryTargetLibrary}::${capabilityId}`;
+        const nextEntry = {
+          ...entry,
+          target_library: entryTargetLibrary,
+          merge: {
+            merged_at: generatedAt,
+            merged_source: bundleSource,
+            merged_source_name: sourceRef.name || null,
+            merged_shard_source: shardSource
+          }
+        };
+        const previous = candidateEntriesByKey.get(key);
+        if (!previous) {
+          candidateEntriesByKey.set(key, nextEntry);
+          continue;
+        }
+        if (comparePublishedAt(previous, nextEntry) <= 0) {
+          superseded.push({
+            capability_id: capabilityId,
+            target_library: entryTargetLibrary,
+            kept_source: bundleSource,
+            dropped_source: normalizeString(previous.merge && previous.merge.merged_source) || null
+          });
+          candidateEntriesByKey.set(key, nextEntry);
+        } else {
+          superseded.push({
+            capability_id: capabilityId,
+            target_library: entryTargetLibrary,
+            kept_source: normalizeString(previous.merge && previous.merge.merged_source) || null,
+            dropped_source: bundleSource
+          });
+        }
+      }
+    }
+    sourceReports.push(sourceReport);
+  }
+
+  let mergedEntries = Array.from(candidateEntriesByKey.values());
+  mergedEntries.sort((left, right) => {
+    const libraryCompare = normalizeString(left.target_library).localeCompare(normalizeString(right.target_library));
+    if (libraryCompare !== 0) {
+      return libraryCompare;
+    }
+    return (Date.parse(right.published_at || '') || 0) - (Date.parse(left.published_at || '') || 0);
+  });
+  if (limit > 0) {
+    mergedEntries = mergedEntries.slice(0, limit);
+  }
+
+  const byLibrary = new Map();
+  for (const entry of mergedEntries) {
+    const targetLibrary = normalizeString(entry.target_library);
+    if (!byLibrary.has(targetLibrary)) {
+      byLibrary.set(targetLibrary, []);
+    }
+    byLibrary.get(targetLibrary).push(entry);
+  }
+
+  const bundleOutFile = normalizeString(options.outFile || options.out_file)
+    || resolveDefaultSemanticSharedMergeBundleOutFile(specId);
+  const indexOutFile = bundleOutFile
+    ? bundleOutFile.replace(/\.bundle\.json$/i, '.index.json')
+    : resolveDefaultSemanticSharedMergeIndexOutFile(specId);
+  const receiptOutFile = bundleOutFile
+    ? bundleOutFile.replace(/\.bundle\.json$/i, '.receipt.json')
+    : resolveDefaultSemanticSharedMergeReceiptOutFile(specId);
+
+  const libraries = [];
+  const indexShards = {};
+  for (const [targetLibrary, entries] of byLibrary.entries()) {
+    const shardFile = specId
+      ? resolveDefaultSemanticSharedMergeShardOutFile(specId, targetLibrary)
+      : '';
+    const shardPayload = {
+      api_version: SEMANTIC_SHARED_SYNC_SHARD_API_VERSION,
+      generated_at: generatedAt,
+      target_library: targetLibrary,
+      total_entries: entries.length,
+      entries
+    };
+    if (shardFile) {
+      const absoluteShardPath = path.join(projectPath, shardFile);
+      await fileSystem.ensureDir(path.dirname(absoluteShardPath));
+      await fileSystem.writeJson(absoluteShardPath, shardPayload, { spaces: 2 });
+    }
+    libraries.push({
+      target_library: targetLibrary,
+      shard_file: shardFile || null,
+      entry_count: entries.length,
+      latest_published_at: normalizeString(entries[0] && entries[0].published_at) || null
+    });
+    indexShards[targetLibrary] = {
+      shard_file: shardFile || null,
+      entry_count: entries.length,
+      latest_published_at: normalizeString(entries[0] && entries[0].published_at) || null
+    };
+  }
+
+  const bundlePayload = {
+    api_version: SEMANTIC_SHARED_SYNC_BUNDLE_API_VERSION,
+    mode: 'semantic-merge-shared',
+    success: blocked.length === 0,
+    generated_at: generatedAt,
+    spec_id: specId || null,
+    mirror_root: '.sce/knowledge/semantic-shared',
+    totals: {
+      sources: sourceRefs.length,
+      candidate_entries: candidateEntries,
+      merged_entries: mergedEntries.length,
+      superseded: superseded.length,
+      blocked: blocked.length
+    },
+    sources: sourceReports,
+    libraries,
+    blocked
+  };
+
+  const indexPayload = {
+    api_version: SEMANTIC_SHARED_SYNC_INDEX_API_VERSION,
+    generated_at: generatedAt,
+    spec_id: specId || null,
+    bundle_file: bundleOutFile || null,
+    shard_count: libraries.length,
+    shards: indexShards
+  };
+
+  const receiptPayload = {
+    api_version: SEMANTIC_SHARED_SYNC_MERGE_RECEIPT_API_VERSION,
+    mode: 'semantic-merge-shared',
+    success: blocked.length === 0,
+    generated_at: generatedAt,
+    spec_id: specId || null,
+    totals: bundlePayload.totals,
+    sources: sourceReports,
+    libraries,
+    superseded,
+    blocked
+  };
+
+  if (bundleOutFile) {
+    const absoluteBundlePath = path.join(projectPath, bundleOutFile);
+    await fileSystem.ensureDir(path.dirname(absoluteBundlePath));
+    await fileSystem.writeJson(absoluteBundlePath, bundlePayload, { spaces: 2 });
+  }
+  if (indexOutFile) {
+    const absoluteIndexPath = path.join(projectPath, indexOutFile);
+    await fileSystem.ensureDir(path.dirname(absoluteIndexPath));
+    await fileSystem.writeJson(absoluteIndexPath, indexPayload, { spaces: 2 });
+  }
+  if (receiptOutFile) {
+    const absoluteReceiptPath = path.join(projectPath, receiptOutFile);
+    await fileSystem.ensureDir(path.dirname(absoluteReceiptPath));
+    await fileSystem.writeJson(absoluteReceiptPath, receiptPayload, { spaces: 2 });
+  }
+
+  return {
+    ...receiptPayload,
+    out_file: bundleOutFile || null,
+    index_file: indexOutFile || null,
+    receipt_file: receiptOutFile || null
+  };
+}
+
+module.exports = {
+  SEMANTIC_SHARED_SYNC_MERGE_RECEIPT_API_VERSION,
+  resolveDefaultSemanticSharedMergeBundleOutFile,
+  resolveDefaultSemanticSharedMergeIndexOutFile,
+  resolveDefaultSemanticSharedMergeReceiptOutFile,
+  resolveDefaultSemanticSharedMergeShardOutFile,
+  mergeSemanticSharedBundles
+};