spec-up-t 1.3.1 → 1.4.0

package/src/{fix-markdown-files.js → pipeline/preprocessing/normalize-terminology-markdown.js}

@@ -1,23 +1,7 @@
 const fs = require('fs');
 const path = require('path');
-const { shouldProcessFile } = require('./utils/file-filter');
-
-/**
- * Checks if a term has a definition starting from the given line index
- * @param {string[]} lines - Array of file lines
- * @param {number} startIndex - Index to start checking from
- * @returns {boolean} - True if definition exists, false otherwise
- */
-function hasDefinition(lines, startIndex) {
-    for (let i = startIndex; i < lines.length; i++) {
-        const line = lines[i].trim();
-        if (line === '') continue; // Skip empty lines
-        if (line.startsWith('~')) return true; // Found definition
-        if (line.startsWith('[[def:') || line.startsWith('[[tref:')) return false; // Found next term
-        if (line.length > 0) return false; // Found other content
-    }
-    return false;
-}
+const { shouldProcessFile } = require('../../utils/file-filter.js');
+const Logger = require('../../utils/logger.js');
 
 /**
  * Handles specific functionality for `[[def:` and `[[tref:` lines
@@ -31,19 +15,41 @@ function processDefLines(lines) {
     for (let i = 0; i < result.length; i++) {
         if (result[i].startsWith('[[def:') || result[i].startsWith('[[tref:')) {
             let insertIndex = i + 1;
-            
+
             // Ensure a blank line immediately follows `[[def:` and `[[tref:` lines
             if (insertIndex < result.length && result[insertIndex].trim() !== '') {
                 result.splice(insertIndex, 0, ''); // Insert blank line
                 insertIndex++;
                 modified = true;
             }
-            
-            // Check if term has a definition
-            if (!hasDefinition(result, insertIndex)) {
-                result.splice(insertIndex, 0, '', '~ No local definition found.', '');
-                modified = true;
-            }
+
+            // No additional content needed - both [[def: and [[tref: lines can exist standalone
+            // The HTML post-processing will handle the proper structure
+        }
+    }
+
+    return { lines: result, modified };
+}
+
+/**
+ * Prepends `~ ` to appropriate lines
+ * @param {string[]} lines - Array of file lines
+ * @returns {object} - Object containing modified lines and modification status
+ */
+function prependTildeToLines(lines) {
+    const result = [...lines];
+    let modified = false;
+
+    for (let i = 0; i < result.length; i++) {
+        if (
+            !result[i].startsWith('[[def:') &&
+            !result[i].startsWith('[[tref:') &&
+            result[i].trim() !== '' &&
+            !result[i].startsWith('~ ') &&
+            !result[i].trim().startsWith('<!--')
+        ) {
+            result[i] = `~ ${result[i]}`;
+            modified = true;
         }
     }
 
@@ -117,6 +123,10 @@ function processMarkdownFile(filePath, fileName) {
         lines = spacingResult.lines;
         modified = modified || spacingResult.modified;
 
+        const tildeResult = prependTildeToLines(lines);
+        lines = tildeResult.lines;
+        modified = modified || tildeResult.modified;
+
         const newlineResult = ensureTrailingNewline(lines);
         lines = newlineResult.lines;
         modified = modified || newlineResult.modified;
@@ -127,7 +137,7 @@ function processMarkdownFile(filePath, fileName) {
             fs.writeFileSync(filePath, newData, 'utf8');
         }
     } catch (err) {
-        console.error(`❌ Error while trying to fix the markdown in file ${fileName}: ${err}`);
+        Logger.error('Error while trying to fix the markdown in file %s: %o', fileName, err);
     }
 }
 
@@ -136,7 +146,7 @@ function processMarkdownFile(filePath, fileName) {
  * @param {string} directory - The directory to process
  * @returns {void}
  */
-function fixMarkdownFiles(directory) {
+function normalizeTerminologyMarkdown(directory) {
     try {
         // Read the contents of the directory synchronously
         const items = fs.readdirSync(directory, { withFileTypes: true });
@@ -145,17 +155,17 @@ function fixMarkdownFiles(directory) {
         items.forEach(item => {
             const itemPath = path.join(directory, item.name);
             if (item.isDirectory()) {
-                // If the item is a directory, call fixMarkdownFiles recursively
-                fixMarkdownFiles(itemPath);
+                // If the item is a directory, call normalizeTerminologyMarkdown recursively
+                normalizeTerminologyMarkdown(itemPath);
             } else if (item.isFile() && shouldProcessFile(item.name)) {
                 processMarkdownFile(itemPath, item.name);
             }
         });
     } catch (err) {
-        console.error(`❌ Error reading directory: ${err}`);
+        Logger.error('Error reading directory: %o', err);
     }
 }
 
 module.exports = {
-    fixMarkdownFiles
+    normalizeTerminologyMarkdown
 };

package/src/pipeline/references/collect-external-references.js

@@ -0,0 +1,307 @@
+/**
+ * @file Orchestrates external reference collection and enrichment within the pipeline.
+ *
+ * This module coordinates three stages:
+ * 1. Scan local markdown for `[[xref:...]]` / `[[tref:...]]` references.
+ * 2. Enrich references with metadata derived from `specs.json`.
+ * 3. Persist the combined dataset for downstream consumers.
+ */
+
+require('dotenv').config();
+const path = require('path');
+const fs = require('fs-extra');
+const readlineSync = require('readline-sync');
+
+const Logger = require('../../utils/logger');
+const { shouldProcessFile } = require('../../utils/file-filter');
+const { getCurrentBranch } = require('../../utils/git-info');
+const { addNewXTrefsFromMarkdown, isXTrefInAnyFile } = require('./xtref-utils');
+const { processXTrefObject } = require('../../parsers/template-tag-parser');
+
+/**
+ * Reuses the main rendering entry point once reference collection has refreshed the cache.
+ * Keeping this invocation on the Node side (instead of the menu script) guarantees that
+ * automated callers of `collectExternalReferences` continue to receive a fully rendered spec.
+ */
+function renderSpecification() {
+    require('../../../index.js')({ nowatch: true });
+}
+
+/**
+ * Normalizes the specs structure pulled from specs.json so callers can rely on predictable shapes.
+ *
+ * The helper guarantees:
+ * - the returned `specs` array is always an array;
+ * - callers get the first spec entry (or an empty object) as `primarySpec`;
+ * - callers receive a safe `externalSpecsRepos` array for follow-up validation or iteration;
+ * - when the specs array is empty, an explanatory error is logged and `null` is returned to signal abortion.
+ *
+ * @param {object} config - Parsed specs configuration.
+ * @param {{ noSpecsMessage: string }} options - Allows callers to tailor the abort message for their context.
+ * @returns {{ specs: Array<object>, primarySpec: object, externalSpecsRepos: Array<object>, hasExternalSpecsField: boolean } | null}
+ */
+function normalizeSpecConfiguration(config, { noSpecsMessage }) {
+    const specs = Array.isArray(config?.specs) ? config.specs : [];
+
+    if (specs.length === 0) {
+        Logger.error(noSpecsMessage);
+        return null;
+    }
+
+    const primarySpec = specs[0] ?? {};
+    const hasExternalSpecsField = Array.isArray(primarySpec.external_specs);
+    const externalSpecsRepos = hasExternalSpecsField ? primarySpec.external_specs : [];
+
+    return { specs, primarySpec, externalSpecsRepos, hasExternalSpecsField };
+}
+
+/**
+ * Augments reference records with repository metadata pulled from `specs.json`.
+ *
+ * @param {object} config - Parsed specs configuration.
+ * @param {Array<object>} xtrefs - Reference entries collected from markdown.
+ */
+function extendXTrefs(config, xtrefs) {
+    if (config.specs[0].external_specs_repos) {
+        Logger.warn('Your specs.json file uses an outdated structure. Update it using: https://github.com/trustoverip/spec-up-t/blob/master/src/install-from-boilerplate/boilerplate/specs.json');
+        return;
+    }
+
+    const repoLookup = new Map();
+    const siteLookup = new Map();
+
+    config.specs.forEach(spec => {
+        spec.external_specs.forEach(repo => {
+            if (repo.external_spec) {
+                repoLookup.set(repo.external_spec, repo);
+            }
+        });
+
+        spec.external_specs
+            .filter(externalSpec => typeof externalSpec === 'object' && externalSpec !== null)
+            .forEach(externalSpec => {
+                const key = Object.keys(externalSpec)[0];
+                siteLookup.set(key, externalSpec[key]);
+            });
+    });
+
+    xtrefs.forEach(xtref => {
+        xtref.repoUrl = null;
+        xtref.terms_dir = null;
+        xtref.owner = null;
+        xtref.repo = null;
+        xtref.site = null;
+        xtref.branch = null;
+
+        const repo = repoLookup.get(xtref.externalSpec);
+        if (repo) {
+            xtref.repoUrl = repo.url;
+            xtref.terms_dir = repo.terms_dir;
+
+            if (xtref.repoUrl) {
+                const urlParts = new URL(xtref.repoUrl).pathname.split('/');
+                xtref.owner = urlParts[1];
+                xtref.repo = urlParts[2];
+            }
+
+            xtref.avatarUrl = repo.avatar_url;
+            xtref.ghPageUrl = repo.gh_page;
+        }
+
+        const site = siteLookup.get(xtref.externalSpec);
+        if (site) {
+            xtref.site = site;
+        }
+
+        try {
+            xtref.branch = getCurrentBranch();
+        } catch (error) {
+            Logger.warn(`Could not get current branch for xtref ${xtref.externalSpec}:${xtref.term}: ${error.message}`);
+            xtref.branch = 'main';
+        }
+    });
+}
+
+/**
+ * Executes the full collection pipeline once pre-flight checks pass.
+ *
+ * @param {object} config - Parsed specs configuration.
+ * @param {string} GITHUB_API_TOKEN - GitHub PAT used for API calls.
+ */
+function processExternalReferences(config, GITHUB_API_TOKEN) {
+    const { processXTrefsData } = require('./process-xtrefs-data');
+    const { doesUrlExist } = require('../../utils/does-url-exist');
+
+    const normalizedConfig = normalizeSpecConfiguration(config, {
+        noSpecsMessage: 'No specs defined in specs.json. Skipping external reference collection.'
+    });
+
+    // Abort collection when the configuration is missing mandatory specs definitions.
+    if (!normalizedConfig) {
+        return;
+    }
+
+    const { specs, externalSpecsRepos } = normalizedConfig;
+
+    externalSpecsRepos.forEach(repo => {
+        doesUrlExist(repo.url)
+            .then(exists => {
+                if (exists) {
+                    return;
+                }
+
+                const userInput = readlineSync.question(
+`❌ This external reference is not a valid URL:
+
+   Repository: ${repo.url},
+   
+   Terms directory: ${repo.terms_dir}
+
+   Please fix the external references in the specs.json file that you will find at the root of your project.
+
+   Do you want to stop? (yes/no): `);
+
+                if (userInput.toLowerCase() === 'yes' || userInput.toLowerCase() === 'y') {
+                    Logger.info('Stopping...');
+                    process.exit(1);
+                }
+            })
+            .catch(error => {
+                Logger.error('Error checking URL existence:', error);
+            });
+    });
+
+    const outputDir = '.cache';
+    const xtrefsHistoryDir = path.join(outputDir, 'xtrefs-history');
+    const outputPathJSON = path.join(outputDir, 'xtrefs-data.json');
+    const outputPathJS = path.join(outputDir, 'xtrefs-data.js');
+    const outputPathJSTimeStamped = path.join(xtrefsHistoryDir, `xtrefs-data-${Date.now()}.js`);
+
+    fs.ensureDirSync(outputDir);
+    fs.ensureDirSync(xtrefsHistoryDir);
+
+    let allXTrefs = { xtrefs: [] };
+    if (fs.existsSync(outputPathJSON)) {
+        const existingXTrefs = fs.readJsonSync(outputPathJSON);
+        if (existingXTrefs?.xtrefs) {
+            allXTrefs = existingXTrefs;
+        }
+    }
+
+    const specTermsDirectories = specs.reduce((directories, spec) => {
+        const specDir = spec?.spec_directory;
+        const termsDir = spec?.spec_terms_directory;
+
+        if (!specDir || !termsDir) {
+            Logger.warn(`Spec entry is missing spec_directory or spec_terms_directory: ${JSON.stringify(spec)}`);
+            return directories;
+        }
+
+        const resolvedDir = path.join(specDir, termsDir);
+
+        if (!fs.existsSync(resolvedDir)) {
+            Logger.warn(`Spec terms directory does not exist: ${resolvedDir}`);
+            return directories;
+        }
+
+        directories.push(resolvedDir);
+        return directories;
+    }, []);
+
+    if (specTermsDirectories.length === 0) {
+        Logger.warn('No spec terms directories found. Skipping external reference collection.');
+        return;
+    }
+
+    const fileContents = new Map();
+
+    specTermsDirectories.forEach(specDirectory => {
+        fs.readdirSync(specDirectory).forEach(file => {
+            if (!shouldProcessFile(file)) {
+                return;
+            }
+
+            const filePath = path.join(specDirectory, file);
+            const markdown = fs.readFileSync(filePath, 'utf8');
+            fileContents.set(file, markdown);
+        });
+    });
+
+    allXTrefs.xtrefs = allXTrefs.xtrefs.filter(existingXTref =>
+        isXTrefInAnyFile(existingXTref, fileContents)
+    );
+
+    fileContents.forEach((content, filename) => {
+        addNewXTrefsFromMarkdown(content, allXTrefs, filename, processXTrefObject);
+    });
+
+    extendXTrefs(config, allXTrefs.xtrefs);
+    return processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, outputPathJS, outputPathJSTimeStamped);
+}
+
+/**
+ * Public entry point for the external reference collection stage.
+ *
+ * @param {{ pat?: string }} options - Optional overrides (GitHub PAT).
+ */
+function collectExternalReferences(options = {}) {
+    const config = fs.readJsonSync('specs.json');
+    const normalizedConfig = normalizeSpecConfiguration(config, {
+        noSpecsMessage: 'No specs defined in specs.json. Nothing to collect.'
+    });
+
+    // Bail out immediately if the specs.json file lacks the required specs collection.
+    if (!normalizedConfig) {
+        return;
+    }
+
+    const { externalSpecsRepos, hasExternalSpecsField } = normalizedConfig;
+    const GITHUB_API_TOKEN = options.pat || process.env.GITHUB_API_TOKEN;
+
+    if (!GITHUB_API_TOKEN) {
+        Logger.warn('No GitHub Personal Access Token (PAT) found. Running without authentication (may hit rate limits).');
+        Logger.info('For better performance, set up a PAT: https://trustoverip.github.io/spec-up-t-website/docs/getting-started/github-token\n');
+    }
+
+    // Communicate that the expected external_specs array is missing entirely.
+    if (!hasExternalSpecsField) {
+        Logger.info(
+            'No external_specs array found on the first spec entry in specs.json. External reference collection is skipped.'
+        );
+        renderSpecification();
+        return;
+    }
+
+    // Let the user know the array exists but contains no repositories, making collection pointless.
+    if (externalSpecsRepos.length === 0) {
+        Logger.info(
+            'The external_specs array in specs.json is empty. Add external repositories to collect external references.'
+        );
+        renderSpecification();
+        return;
+    }
+
+    const pipeline = processExternalReferences(config, GITHUB_API_TOKEN);
+
+    // If the pipeline short-circuited (e.g. missing configuration), render immediately and return its value.
+    if (pipeline && typeof pipeline.then === 'function') {
+        return pipeline
+            .then(result => {
+                renderSpecification();
+                return result;
+            })
+            .catch(error => {
+                Logger.error('Rendering failed after collecting external references.', error);
+                throw error;
+            });
+    } else {
+        renderSpecification();
+        return pipeline;
+    }
+}
+
+module.exports = {
+    collectExternalReferences,
+    extendXTrefs,
+    processExternalReferences
+};

package/src/pipeline/references/external-references-service.js

@@ -0,0 +1,231 @@
+const cheerio = require("cheerio");
+const axios = require('axios').default;
+const fs = require('fs-extra');
+const Logger = require('../../utils/logger.js');
+
+const spaceRegex = /\s+/g;
+
+function validateReferences(references, definitions, render) {
+  const unresolvedRefs = [];
+  [...new Set(references)].forEach(
+    ref => {
+      if (render.includes(`id="term:${ref.replace(spaceRegex, '-').toLowerCase()}"`)) {
+        // Reference is resolved
+      } else {
+        unresolvedRefs.push(ref);
+      }
+    }
+  );
+  if (unresolvedRefs.length > 0) {
+    Logger.info('Unresolved References:', unresolvedRefs);
+  }
+
+  const danglingDefs = [];
+  definitions.forEach(def => {
+    // Handle both old array format and new object format
+    if (Array.isArray(def)) {
+      let found = def.some(term => render.includes(`href="#term:${term.replace(spaceRegex, '-').toLowerCase()}"`))
+      if (!found) {
+        danglingDefs.push(def[0]);
+      }
+    } else if (def.term) {
+      // New object format
+      const terms = [def.term, def.alias].filter(Boolean);
+      let found = terms.some(term => render.includes(`href="#term:${term.replace(spaceRegex, '-').toLowerCase()}"`))
+      if (!found) {
+        danglingDefs.push(def.term);
+      }
+    }
+  })
+  if (danglingDefs.length > 0) {
+    Logger.info('Dangling Definitions:', danglingDefs);
+  }
+}
+
+function findExternalSpecByKey(config, key) {
+  if (!config || !config.specs) return null;
+  for (const spec of config.specs) {
+    if (spec.external_specs) {
+      for (const externalSpec of spec.external_specs) {
+        if (externalSpec.external_spec === key) {
+          return externalSpec;
+        }
+      }
+    }
+  }
+  return null;
+}
+
+async function fetchExternalSpecs(spec) {
+  try {
+    let results = await Promise.all(
+      spec.external_specs.map(s => {
+        const url = s["gh_page"];
+        return axios.get(url).catch(error => ({ error, url }));
+      })
+    );
+
+    const failed = results.filter(r => r && r.error);
+    if (failed.length > 0) {
+      failed.forEach(f => {
+        const msg = f.error.response
+          ? `HTTP ${f.error.response.status} for ${f.url}`
+          : `Network error for ${f.url}: ${f.error.message}`;
+        Logger.error("External spec fetch failed:", msg);
+      });
+    }
+
+    // Map results to extract terms instead of creating DOM HTML
+    const extractedTerms = [];
+
+    results
+      .map((r, index) =>
+        r && r.status === 200
+          ? { externalSpec: spec.external_specs[index].external_spec, data: r.data }
+          : null
+      )
+      .filter(r => r) // Remove null values (failed fetches)
+      .forEach(r => {
+        // Extract terms from each external spec's HTML
+        const termsFromSpec = extractTermsFromHtml(r.externalSpec, r.data);
+        extractedTerms.push(...termsFromSpec);
+      });
+
+    return extractedTerms;
+  } catch (e) {
+    Logger.error("Unexpected error in fetchExternalSpecs:", e);
+    return [];
+  }
+}
+
+
+/**
+ * Merges xref terms from external specs into the allXTrefs structure
+ * @param {Array} xrefTerms - Array of xref term objects from fetchExternalSpecs
+ * @param {string} outputPathJSON - Path to the xtrefs-data.json file
+ * @param {string} outputPathJS - Path to the xtrefs-data.js file
+ * @returns {Promise<void>}
+ */
+async function mergeXrefTermsIntoAllXTrefs(xrefTerms, outputPathJSON, outputPathJS) {
+  try {
+    let allXTrefs = { xtrefs: [] };
+
+    // Load existing xtrefs data if it exists
+    if (fs.existsSync(outputPathJSON)) {
+      allXTrefs = fs.readJsonSync(outputPathJSON);
+    }
+
+    // Add xref terms to the allXTrefs structure
+    // Mark them with source: 'xref' to distinguish from tref entries
+    xrefTerms.forEach(xrefTerm => {
+      // Check if this term already exists (avoid duplicates)
+      const existingIndex = allXTrefs.xtrefs.findIndex(existing =>
+        existing.externalSpec === xrefTerm.externalSpec &&
+        existing.term === xrefTerm.term &&
+        existing.source === 'xref'
+      );
+
+      if (existingIndex >= 0) {
+        // Update existing entry
+        allXTrefs.xtrefs[existingIndex] = {
+          ...allXTrefs.xtrefs[existingIndex],
+          ...xrefTerm,
+          lastUpdated: new Date().toISOString()
+        };
+      } else {
+        // Add new entry
+        allXTrefs.xtrefs.push({
+          ...xrefTerm,
+          lastUpdated: new Date().toISOString()
+        });
+      }
+    });
+
+    // Write the updated data back to files
+    const allXTrefsStr = JSON.stringify(allXTrefs, null, 2);
+    fs.writeFileSync(outputPathJSON, allXTrefsStr, 'utf8');
+
+    const stringReadyForFileWrite = `const allXTrefs = ${allXTrefsStr};`;
+    fs.writeFileSync(outputPathJS, stringReadyForFileWrite, 'utf8');
+
+    Logger.success(`Merged ${xrefTerms.length} xref terms into allXTrefs. Total entries: ${allXTrefs.xtrefs.length}`);
+
+  } catch (error) {
+    Logger.error('Error merging xref terms into allXTrefs:', error.message);
+  }
+}
+
+/**
+ * Extracts terms and their definitions from HTML and returns them as structured data
+ * @param {string} externalSpec - The external spec identifier
+ * @param {string} html - The HTML content to parse
+ * @returns {Array} Array of term objects suitable for the allXTrefs structure
+ */
+function extractTermsFromHtml(externalSpec, html) {
+  try {
+    const $ = cheerio.load(html);
+    const terms = [];
+
+    const termElements = $('dl.terms-and-definitions-list dt');
+    Logger.highlight(`Found ${termElements.length} term elements in ${externalSpec} (HTML size: ${Math.round(html.length / 1024)}KB)`);
+
+    // Process terms in batches to prevent stack overflow with large datasets
+    const BATCH_SIZE = 100;
+    const totalElements = termElements.length;
+
+    for (let i = 0; i < totalElements; i += BATCH_SIZE) {
+      const batch = termElements.slice(i, i + BATCH_SIZE);
+
+      batch.each((index, termElement) => {
+        try {
+          const $termElement = $(termElement);
+          const termId = $termElement.attr('id');
+          
+          // Skip elements without an id attribute or with invalid id format
+          if (!termId || !termId.includes('term:')) {
+            return;
+          }
+          
+          const termName = termId.replace('term:', '');
+          const dd = $termElement.next('dd');
+
+          if (dd.length > 0) {
+            // Create term object compatible with allXTrefs structure
+            const termObj = {
+              externalSpec: externalSpec,
+              term: termName,
+              content: $.html(dd), // Store the complete DD content
+              // Add metadata for consistency with tref structure
+              source: 'xref', // Distinguish from tref entries
+              termId: `term:${externalSpec}:${termName}`, // Fully qualified term ID
+            };
+
+            terms.push(termObj);
+          }
+        } catch (termError) {
+          Logger.warn(`Error processing term in ${externalSpec}:`, termError.message);
+        }
+      });
+
+      // Log progress for very large datasets
+      if (totalElements > 1000 && i % (BATCH_SIZE * 10) === 0) {
+        Logger.progress(Math.min(i + BATCH_SIZE, totalElements), totalElements, `Processing terms from ${externalSpec}`);
+      }
+    }
+
+    Logger.success(`Extracted ${terms.length} terms from external spec: ${externalSpec}`);
+    return terms;
+
+  } catch (error) {
+    Logger.error(`Error extracting terms from external spec '${externalSpec}':`, error.message);
+    return [];
+  }
+}
+
+module.exports = {
+  findExternalSpecByKey,
+  validateReferences,
+  fetchExternalSpecs,
+  extractTermsFromHtml,
+  mergeXrefTermsIntoAllXTrefs
+}