scene-capability-engine 3.6.64 → 3.6.67

package/lib/semantic/shared-sync-intake.js

@@ -0,0 +1,592 @@
+const path = require('path');
+const http = require('http');
+const https = require('https');
+const fs = require('fs-extra');
+const {
+  SHARED_LIBRARY_ROOT,
+  resolveSharedLibraryEntryRelativePath,
+  upsertSharedLibraryIndex
+} = require('./shared-publication');
+
+const SEMANTIC_SHARED_SOURCE_CONFIG = '.sce/config/semantic-shared-sources.json';
+const SEMANTIC_SHARED_SYNC_INTAKE_API_VERSION = 'sce.semantic.shared-sync.intake/v0.1';
+const SEMANTIC_SHARED_SYNC_BUNDLE_API_VERSION = 'sce.semantic.shared-sync.bundle/v0.1';
+const SEMANTIC_SHARED_SYNC_SHARD_API_VERSION = 'sce.semantic.shared-sync.shard/v0.1';
+
+function normalizeString(value) {
+  if (typeof value !== 'string') {
+    return '';
+  }
+  return value.trim();
+}
+
+function normalizeInteger(value, fallback = 0) {
+  const parsed = Number.parseInt(`${value}`, 10);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    return fallback;
+  }
+  return parsed;
+}
+
+function normalizeBoolean(value, fallback = false) {
+  if (typeof value === 'boolean') {
+    return value;
+  }
+  const normalized = normalizeString(`${value || ''}`).toLowerCase();
+  if (!normalized) {
+    return fallback;
+  }
+  if (['1', 'true', 'yes', 'y', 'on'].includes(normalized)) {
+    return true;
+  }
+  if (['0', 'false', 'no', 'n', 'off'].includes(normalized)) {
+    return false;
+  }
+  return fallback;
+}
+
+function isObject(value) {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}
+
+function toProjectRelative(projectPath, targetPath) {
+  return path.relative(projectPath, targetPath).replace(/\\/g, '/');
+}
+
+function resolveSemanticSharedSourceConfigPath(projectPath = process.cwd(), configPath = '') {
+  const normalized = normalizeString(configPath) || SEMANTIC_SHARED_SOURCE_CONFIG;
+  return path.isAbsolute(normalized)
+    ? normalized
+    : path.resolve(projectPath, normalized);
+}
+
+function resolveDefaultSemanticSharedSyncIntakeOutFile(specId = '') {
+  const normalizedSpecId = normalizeString(specId);
+  if (!normalizedSpecId) {
+    return '';
+  }
+  return `.sce/specs/${normalizedSpecId}/intake/semantic-shared/latest.json`;
+}
+
+function isHttpSource(source = '') {
+  return /^https?:\/\//i.test(normalizeString(source));
+}
+
+function fetchJsonFromHttp(source, timeoutMs = 15000) {
+  const normalized = normalizeString(source);
+  if (!normalized) {
+    return Promise.reject(new Error('shared sync source is required'));
+  }
+  const client = normalized.startsWith('https://') ? https : http;
+  return new Promise((resolve, reject) => {
+    const request = client.get(normalized, {
+      timeout: timeoutMs,
+      headers: {
+        Accept: 'application/json'
+      }
+    }, (response) => {
+      const chunks = [];
+      response.on('data', (chunk) => chunks.push(chunk));
+      response.on('end', () => {
+        const body = Buffer.concat(chunks).toString('utf8');
+        if (response.statusCode < 200 || response.statusCode >= 300) {
+          reject(new Error(`shared sync source responded ${response.statusCode}`));
+          return;
+        }
+        try {
+          resolve(JSON.parse(body));
+        } catch (error) {
+          reject(new Error(`shared sync source returned invalid JSON: ${error.message}`));
+        }
+      });
+    });
+    request.on('timeout', () => {
+      request.destroy(new Error('shared sync source request timed out'));
+    });
+    request.on('error', reject);
+  });
+}
+
+async function loadJsonSource(projectPath, source, fileSystem = fs) {
+  const normalized = normalizeString(source);
+  if (!normalized) {
+    throw new Error('shared sync source is required');
+  }
+  if (isHttpSource(normalized)) {
+    return fetchJsonFromHttp(normalized);
+  }
+  const absolutePath = path.isAbsolute(normalized)
+    ? normalized
+    : path.resolve(projectPath, normalized);
+  if (!await fileSystem.pathExists(absolutePath)) {
+    throw new Error(`shared sync source file not found: ${source}`);
+  }
+  return fileSystem.readJson(absolutePath);
+}
+
+async function readSemanticSharedSourceConfig(projectPath, options = {}, fileSystem = fs) {
+  const configFile = resolveSemanticSharedSourceConfigPath(projectPath, options.config);
+  if (!await fileSystem.pathExists(configFile)) {
+    return {
+      enabled: true,
+      mirror_root: SHARED_LIBRARY_ROOT,
+      sources: []
+    };
+  }
+  const payload = await fileSystem.readJson(configFile);
+  if (!isObject(payload)) {
+    throw new Error(`semantic shared source config must be a JSON object: ${configFile}`);
+  }
+  return payload;
+}
+
+function inferSourceRootFromSource(source = '') {
+  const normalized = normalizeString(source);
+  if (!normalized) {
+    return '';
+  }
+  if (isHttpSource(normalized)) {
+    const sceMarkerIndex = normalized.indexOf('/.sce/');
+    if (sceMarkerIndex > 0) {
+      return normalized.slice(0, sceMarkerIndex);
+    }
+    try {
+      const parsed = new URL(normalized);
+      const pathname = parsed.pathname || '';
+      const markerIndex = pathname.indexOf('/.sce/');
+      if (markerIndex > -1) {
+        parsed.pathname = pathname.slice(0, markerIndex) || '/';
+        parsed.search = '';
+        parsed.hash = '';
+        return parsed.toString().replace(/\/$/, '');
+      }
+    } catch (_error) {
+      return '';
+    }
+    return normalizeString(normalized.replace(/\/[^/]*$/, ''));
+  }
+
+  const normalizedPath = path.resolve(normalized);
+  const marker = `${path.sep}.sce${path.sep}`;
+  const markerIndex = normalizedPath.indexOf(marker);
+  if (markerIndex > -1) {
+    return normalizedPath.slice(0, markerIndex);
+  }
+  return path.dirname(normalizedPath);
+}
+
+function joinSourceRoot(sourceRoot = '', relativePath = '') {
+  const normalizedRoot = normalizeString(sourceRoot);
+  const normalizedRelativePath = normalizeString(relativePath).replace(/\\/g, '/').replace(/^\/+/, '');
+  if (!normalizedRoot || !normalizedRelativePath) {
+    return '';
+  }
+  if (isHttpSource(normalizedRoot)) {
+    return `${normalizedRoot.replace(/\/+$/, '')}/${normalizedRelativePath}`;
+  }
+  return path.resolve(normalizedRoot, normalizedRelativePath);
+}
+
+function resolveShardSource(sourceRoot = '', shardFile = '', bundleSource = '') {
+  const normalizedShardFile = normalizeString(shardFile);
+  if (!normalizedShardFile) {
+    return '';
+  }
+  if (isHttpSource(normalizedShardFile) || path.isAbsolute(normalizedShardFile)) {
+    return normalizedShardFile;
+  }
+  const normalizedRoot = normalizeString(sourceRoot) || inferSourceRootFromSource(bundleSource);
+  if (normalizedRoot) {
+    return joinSourceRoot(normalizedRoot, normalizedShardFile);
+  }
+  if (isHttpSource(bundleSource)) {
+    return new URL(normalizedShardFile, bundleSource).toString();
+  }
+  return path.resolve(path.dirname(bundleSource), normalizedShardFile);
+}
+
+function normalizeSourceCandidate(candidate = {}) {
+  if (!isObject(candidate)) {
+    return null;
+  }
+  const bundle = normalizeString(candidate.bundle || candidate.source || candidate.url || candidate.file);
+  if (!bundle) {
+    return null;
+  }
+  return {
+    name: normalizeString(candidate.name) || 'semantic-shared',
+    enabled: normalizeBoolean(candidate.enabled, true),
+    bundle,
+    root: normalizeString(candidate.root || candidate.base || candidate.base_url || candidate.base_path),
+    required_release_state: normalizeString(candidate.required_release_state || candidate.requiredReleaseState)
+  };
+}
+
+function resolveRequiredReleaseState(options = {}, configuredSource = null) {
+  return normalizeString(options.requiredReleaseState || options.required_release_state)
+    || normalizeString(configuredSource && configuredSource.required_release_state);
+}
+
+function readBundleReleaseState(bundlePayload = {}) {
+  if (!isObject(bundlePayload)) {
+    return '';
+  }
+  if (isObject(bundlePayload.governance)) {
+    return normalizeString(
+      bundlePayload.governance.release_state
+      || bundlePayload.governance.publish_state
+      || bundlePayload.governance.state
+    );
+  }
+  return normalizeString(bundlePayload.release_state);
+}
+
+function resolveConfiguredSemanticSource(config = {}, requestedSourceName = '') {
+  const sources = Array.isArray(config.sources)
+    ? config.sources.map((item) => normalizeSourceCandidate(item)).filter(Boolean)
+    : [];
+  if (!sources.length) {
+    return null;
+  }
+  const requestedName = normalizeString(requestedSourceName);
+  if (requestedName) {
+    return sources.find((item) => item.enabled && item.name === requestedName) || null;
+  }
+  return sources.find((item) => item.enabled) || null;
+}
+
+async function readJsonIfExists(fileSystem, absolutePath, fallback) {
+  if (!await fileSystem.pathExists(absolutePath)) {
+    return fallback;
+  }
+  try {
+    return await fileSystem.readJson(absolutePath);
+  } catch (_error) {
+    return fallback;
+  }
+}
+
+function comparePublishedAt(left = {}, right = {}) {
+  const leftTime = Date.parse(left.published_at || '') || 0;
+  const rightTime = Date.parse(right.published_at || '') || 0;
+  return leftTime - rightTime;
+}
+
+async function syncSemanticSharedBundle(options = {}, dependencies = {}) {
+  const projectPath = dependencies.projectPath || process.cwd();
+  const fileSystem = dependencies.fileSystem || fs;
+  const config = await readSemanticSharedSourceConfig(projectPath, {
+    config: options.config
+  }, fileSystem);
+  if (normalizeBoolean(config.enabled, true) === false) {
+    throw new Error('semantic shared source config is disabled');
+  }
+
+  const configuredSource = resolveConfiguredSemanticSource(config, options.sourceName);
+  const source = normalizeString(options.source)
+    || normalizeString(configuredSource && configuredSource.bundle);
+  if (!source) {
+    throw new Error('shared sync source is required (use --source or configure .sce/config/semantic-shared-sources.json)');
+  }
+  const sourceName = normalizeString(options.sourceName)
+    || normalizeString(configuredSource && configuredSource.name)
+    || 'semantic-shared';
+  const sourceRoot = normalizeString(options.sourceRoot)
+    || normalizeString(configuredSource && configuredSource.root)
+    || inferSourceRootFromSource(source);
+  const requiredReleaseState = resolveRequiredReleaseState(options, configuredSource);
+
+  const bundlePayload = await loadJsonSource(projectPath, source, fileSystem);
+  if (!isObject(bundlePayload) || normalizeString(bundlePayload.api_version) !== SEMANTIC_SHARED_SYNC_BUNDLE_API_VERSION) {
+    throw new Error(`invalid semantic shared sync bundle: ${source}`);
+  }
+
+  const bundleLibraries = Array.isArray(bundlePayload.libraries) ? bundlePayload.libraries : [];
+  const specId = normalizeString(options.spec)
+    || normalizeString(bundlePayload.spec_id);
+  const limit = normalizeInteger(options.limit, 100);
+  const generatedAt = new Date().toISOString();
+  const releaseState = readBundleReleaseState(bundlePayload) || null;
+  const imported = [];
+  const skipped = [];
+  const blocked = [];
+
+  if (requiredReleaseState && releaseState !== requiredReleaseState) {
+    blocked.push({
+      reason: 'release-state-mismatch',
+      required_release_state: requiredReleaseState,
+      actual_release_state: releaseState
+    });
+  }
+
+  if (blocked.length === 0) {
+    for (const library of bundleLibraries.slice(0, limit > 0 ? limit : bundleLibraries.length)) {
+      const targetLibrary = normalizeString(library && library.target_library);
+      const shardFile = normalizeString(library && library.shard_file);
+      const shardSource = resolveShardSource(sourceRoot, shardFile, source);
+      if (!targetLibrary || !shardSource) {
+        blocked.push({
+          target_library: targetLibrary || null,
+          reason: 'missing-target-library-or-shard-source',
+          shard_file: shardFile || null
+        });
+        continue;
+      }
+
+      let shardPayload;
+      try {
+        shardPayload = await loadJsonSource(projectPath, shardSource, fileSystem);
+      } catch (error) {
+        blocked.push({
+          target_library: targetLibrary,
+          reason: 'failed-to-load-shard',
+          shard_file: shardFile || null,
+          shard_source: shardSource,
+          detail: error.message
+        });
+        continue;
+      }
+      if (!isObject(shardPayload) || normalizeString(shardPayload.api_version) !== SEMANTIC_SHARED_SYNC_SHARD_API_VERSION) {
+        blocked.push({
+          target_library: targetLibrary,
+          reason: 'invalid-shard-payload',
+          shard_file: shardFile || null,
+          shard_source: shardSource
+        });
+        continue;
+      }
+
+      const entries = Array.isArray(shardPayload.entries) ? shardPayload.entries : [];
+      for (const entry of entries) {
+        const capabilityId = normalizeString(entry && entry.capability_id);
+        const lessonId = normalizeString(entry && entry.lesson_id);
+        const normalizedTargetLibrary = normalizeString(entry && entry.target_library) || targetLibrary;
+        if (!capabilityId || !normalizedTargetLibrary) {
+          blocked.push({
+            capability_id: capabilityId || null,
+            lesson_id: lessonId || null,
+            target_library: normalizedTargetLibrary || null,
+            reason: 'missing-capability-or-library'
+          });
+          continue;
+        }
+
+        const entryRelativePath = resolveSharedLibraryEntryRelativePath(normalizedTargetLibrary, capabilityId);
+        const entryAbsolutePath = path.join(projectPath, entryRelativePath);
+        const existingEntry = await readJsonIfExists(fileSystem, entryAbsolutePath, null);
+        if (existingEntry && comparePublishedAt(existingEntry, entry) >= 0) {
+          skipped.push({
+            capability_id: capabilityId,
+            lesson_id: lessonId || null,
+            target_library: normalizedTargetLibrary,
+            reason: 'stale-or-duplicate'
+          });
+          continue;
+        }
+
+        const nextEntry = {
+          ...entry,
+          target_library: normalizedTargetLibrary,
+          intake: {
+            imported_at: generatedAt,
+            source_name: sourceName,
+            source_bundle: source,
+            source_root: sourceRoot || null,
+            source_shard: shardSource,
+            source_shard_file: shardFile || null
+          }
+        };
+
+        await fileSystem.ensureDir(path.dirname(entryAbsolutePath));
+        await fileSystem.writeJson(entryAbsolutePath, nextEntry, { spaces: 2 });
+        const indexPointers = await upsertSharedLibraryIndex(normalizedTargetLibrary, {
+          capability_id: capabilityId,
+          lesson_id: lessonId || null,
+          project_id: normalizeString(nextEntry.project_id) || null,
+          channel_id: normalizeString(nextEntry.channel_id) || null,
+          session_id: normalizeString(nextEntry.session_id) || null,
+          scene_id: normalizeString(nextEntry.scene_id) || null,
+          spec_id: normalizeString(nextEntry.spec_id) || null,
+          published_at: normalizeString(nextEntry.published_at) || generatedAt,
+          entry_file: entryRelativePath,
+          source_manifest: source,
+          receipt_id: normalizeString(nextEntry.receipt_id) || null
+        }, {
+          projectPath,
+          fileSystem
+        });
+
+        imported.push({
+          capability_id: capabilityId,
+          lesson_id: lessonId || null,
+          target_library: normalizedTargetLibrary,
+          entry_file: entryRelativePath,
+          index_file: indexPointers.library_index_file,
+          shard_source: shardSource
+        });
+      }
+    }
+  }
+
+  const payload = {
+    api_version: SEMANTIC_SHARED_SYNC_INTAKE_API_VERSION,
+    mode: 'semantic-sync-shared',
+    success: blocked.length === 0,
+    generated_at: generatedAt,
+    source,
+    source_name: sourceName,
+    source_root: sourceRoot || null,
+    release_state: releaseState,
+    required_release_state: requiredReleaseState || null,
+    release_state_ok: requiredReleaseState ? releaseState === requiredReleaseState : true,
+    spec_id: specId || null,
+    mirror_root: normalizeString(config.mirror_root) || SHARED_LIBRARY_ROOT,
+    totals: {
+      libraries: bundleLibraries.length,
+      imported: imported.length,
+      skipped: skipped.length,
+      blocked: blocked.length
+    },
+    imported,
+    skipped,
+    blocked
+  };
+
+  const outFile = normalizeString(options.outFile || options.out_file)
+    || resolveDefaultSemanticSharedSyncIntakeOutFile(specId);
+  if (outFile) {
+    const absoluteOutPath = path.isAbsolute(outFile)
+      ? outFile
+      : path.join(projectPath, outFile);
+    await fileSystem.ensureDir(path.dirname(absoluteOutPath));
+    await fileSystem.writeJson(absoluteOutPath, payload, { spaces: 2 });
+    payload.out_file = toProjectRelative(projectPath, absoluteOutPath);
+  }
+
+  return payload;
+}
+
+async function inspectSemanticSharedSource(options = {}, dependencies = {}) {
+  const projectPath = dependencies.projectPath || process.cwd();
+  const fileSystem = dependencies.fileSystem || fs;
+  const config = await readSemanticSharedSourceConfig(projectPath, {
+    config: options.config
+  }, fileSystem);
+  const warnings = [];
+  const errors = [];
+  if (normalizeBoolean(config.enabled, true) === false) {
+    warnings.push('semantic shared source config is disabled');
+  }
+
+  const configuredSource = resolveConfiguredSemanticSource(config, options.sourceName);
+  const source = normalizeString(options.source)
+    || normalizeString(configuredSource && configuredSource.bundle);
+  if (!source) {
+    errors.push('shared sync source is required');
+    return {
+      api_version: SEMANTIC_SHARED_SYNC_INTAKE_API_VERSION,
+      mode: 'semantic-health-shared-source',
+      success: false,
+      source: null,
+      source_name: normalizeString(options.sourceName) || null,
+      bundle_ok: false,
+      shard_sources_checked: 0,
+      shard_sources_ok: 0,
+      total_entries: 0,
+      warnings,
+      errors
+    };
+  }
+  const sourceName = normalizeString(options.sourceName)
+    || normalizeString(configuredSource && configuredSource.name)
+    || 'semantic-shared';
+  const sourceRoot = normalizeString(options.sourceRoot)
+    || normalizeString(configuredSource && configuredSource.root)
+    || inferSourceRootFromSource(source);
+  const requiredReleaseState = resolveRequiredReleaseState(options, configuredSource);
+
+  let bundlePayload = null;
+  let bundleOk = false;
+  let totalEntries = 0;
+  let releaseState = null;
+  let releaseStateOk = true;
+  const shardReports = [];
+  try {
+    bundlePayload = await loadJsonSource(projectPath, source, fileSystem);
+    if (!isObject(bundlePayload) || normalizeString(bundlePayload.api_version) !== SEMANTIC_SHARED_SYNC_BUNDLE_API_VERSION) {
+      errors.push(`invalid semantic shared sync bundle: ${source}`);
+    } else {
+      bundleOk = true;
+      releaseState = readBundleReleaseState(bundlePayload) || null;
+      if (requiredReleaseState && releaseState !== requiredReleaseState) {
+        releaseStateOk = false;
+        errors.push(`required release state ${requiredReleaseState} but got ${releaseState || 'unreleased'}`);
+      }
+    }
+  } catch (error) {
+    errors.push(`failed to load bundle (${source}): ${error.message}`);
+  }
+
+  if (bundleOk) {
+    const bundleLibraries = Array.isArray(bundlePayload.libraries) ? bundlePayload.libraries : [];
+    for (const library of bundleLibraries) {
+      const targetLibrary = normalizeString(library && library.target_library);
+      const shardFile = normalizeString(library && library.shard_file);
+      const shardSource = resolveShardSource(sourceRoot, shardFile, source);
+      const report = {
+        target_library: targetLibrary || null,
+        shard_file: shardFile || null,
+        shard_source: shardSource || null,
+        ok: false,
+        entries: 0,
+        error: null
+      };
+      if (!targetLibrary || !shardSource) {
+        report.error = 'missing-target-library-or-shard-source';
+        shardReports.push(report);
+        continue;
+      }
+      try {
+        const shardPayload = await loadJsonSource(projectPath, shardSource, fileSystem);
+        if (!isObject(shardPayload) || normalizeString(shardPayload.api_version) !== SEMANTIC_SHARED_SYNC_SHARD_API_VERSION) {
+          report.error = 'invalid-shard-payload';
+        } else {
+          report.ok = true;
+          report.entries = Array.isArray(shardPayload.entries) ? shardPayload.entries.length : 0;
+          totalEntries += report.entries;
+        }
+      } catch (error) {
+        report.error = error.message;
+      }
+      shardReports.push(report);
+    }
+  }
+
+  return {
+    api_version: SEMANTIC_SHARED_SYNC_INTAKE_API_VERSION,
+    mode: 'semantic-health-shared-source',
+    success: errors.length === 0 && shardReports.every((item) => item.ok),
+    source,
+    source_name: sourceName,
+    source_root: sourceRoot || null,
+    release_state: releaseState,
+    required_release_state: requiredReleaseState || null,
+    release_state_ok: releaseStateOk,
+    bundle_ok: bundleOk,
+    shard_sources_checked: shardReports.length,
+    shard_sources_ok: shardReports.filter((item) => item.ok).length,
+    total_entries: totalEntries,
+    warnings,
+    errors,
+    shards: shardReports
+  };
+}
+
+module.exports = {
+  SEMANTIC_SHARED_SOURCE_CONFIG,
+  SEMANTIC_SHARED_SYNC_INTAKE_API_VERSION,
+  resolveSemanticSharedSourceConfigPath,
+  resolveDefaultSemanticSharedSyncIntakeOutFile,
+  syncSemanticSharedBundle,
+  inspectSemanticSharedSource
+};