spec-up-t 1.3.1 → 1.4.0

package/src/{create-term-index.js → pipeline/configuration/create-term-index.js}

@@ -12,13 +12,14 @@
  *
  * @requires fs-extra - File system operations with extra methods.
  * @requires path - Utilities for working with file and directory paths.
- * @file src/create-term-index.js
+ * @file src/pipeline/configuration/create-term-index.js
  * @author Kor Dwarshuis
  * @version 1.1.0
  * @since 2024-09-02
  */
 
-const { shouldProcessFile } = require('./utils/file-filter');
+const { shouldProcessFile } = require('../../utils/file-filter.js');
+const Logger = require('../../utils/logger.js');
 
 function createTermIndex() {
     try {
@@ -28,41 +29,41 @@ function createTermIndex() {
         
         // Check if specs.json exists
         if (!fs.existsSync(configPath)) {
-            console.warn(`Config file '${configPath}' not found. Using default configuration.`);
+            Logger.warn(`Config file '${configPath}' not found. Using default configuration.`);
             var config = { specs: [] };
         } else {
             // Read config with try-catch to handle parsing errors
             try {
                 var config = fs.readJsonSync(configPath);
             } catch (readError) {
-                console.warn(`Error reading config file: ${readError.message}. Using default configuration.`);
+                Logger.warn(`Error reading config file: ${readError.message}. Using default configuration.`);
                 var config = { specs: [] };
             }
         }
         
         // Provide defaults for missing config
         if (!config) {
-            console.warn('Config file is empty or invalid. Using defaults.');
+            Logger.warn('Config file is empty or invalid. Using defaults.');
             config = { specs: [] };
         }
         
         if (!config.specs) {
-            console.warn('No specs array found in config. Creating empty specs array.');
+            Logger.warn('No specs array found in config. Creating empty specs array.');
             config.specs = [];
         } else if (!Array.isArray(config.specs)) {
-            console.warn('Config specs is not an array. Converting to array.');
+            Logger.warn('Config specs is not an array. Converting to array.');
             config.specs = [config.specs]; // Convert to array if it's an object
         }
         
         // If no valid specs, create an empty term index
         if (config.specs.length === 0) {
-            console.warn('No specs found in configuration. Creating an empty term index.');
+            Logger.warn('No specs found in configuration. Creating an empty term index.');
         }
         
         // Extract spec directories with fallback to current directory
         const specDirectories = config.specs.map((spec, index) => {
             if (!spec.spec_directory) {
-                console.warn(`Warning: spec_directory missing in specs.json entry #${index + 1}. Using current directory.`);
+                Logger.warn(`spec_directory missing in specs.json entry #${index + 1}. Using current directory.`);
                 return '.';  // Default to current directory
             }
             return spec.spec_directory;
@@ -71,7 +72,7 @@ function createTermIndex() {
         // Extract term directories with fallback to default value
         const specTermDirectoryName = config.specs.map((spec, index) => {
             if (!spec.spec_terms_directory) {
-                console.warn(`Warning: spec_terms_directory missing in specs.json entry #${index + 1}. Using default 'terms' directory.`);
+                Logger.warn(`spec_terms_directory missing in specs.json entry #${index + 1}. Using default 'terms' directory.`);
                 return 'terms'; // Default directory name for terms
             }
             return spec.spec_terms_directory;
@@ -79,7 +80,7 @@ function createTermIndex() {
         
         // Safety check - if we have no valid entries, warn and exit cleanly
         if (specDirectories.length === 0 || specTermDirectoryName.length === 0) {
-            console.log('No term directories found in configuration. Creating empty term index.');
+            Logger.info('No term directories found in configuration. Creating empty term index.');
             
             // Create an empty term index
             const outputPathJSON = path.join('.cache', 'term-index.json');
@@ -87,14 +88,14 @@ function createTermIndex() {
                 fs.mkdirSync('.cache', { recursive: true });
             }
             fs.writeJsonSync(outputPathJSON, [], { spaces: 2 });
-            console.log(`✅ Empty term index created at: ${outputPathJSON}`);
+            Logger.success(`Empty term index created at: ${outputPathJSON}`);
             return; // Exit function early
         }
         
         // Verify that the base spec directory exists
         const baseSpecDir = specDirectories[0];
         if (!baseSpecDir || !fs.existsSync(baseSpecDir)) {
-            console.warn(`Spec directory '${baseSpecDir}' does not exist. Creating empty term index.`);
+            Logger.warn(`Spec directory '${baseSpecDir}' does not exist. Creating empty term index.`);
             
             // Create an empty term index
             const outputPathJSON = path.join('.cache', 'term-index.json');
@@ -102,7 +103,7 @@ function createTermIndex() {
                 fs.mkdirSync('.cache', { recursive: true });
             }
             fs.writeJsonSync(outputPathJSON, [], { spaces: 2 });
-            console.log(`✅ Empty term index created at: ${outputPathJSON}`);
+            Logger.success(`Empty term index created at: ${outputPathJSON}`);
             return; // Exit function early
         }
         
@@ -111,14 +112,14 @@ function createTermIndex() {
         
         let files = [];
         if (!fs.existsSync(termsDir)) {
-            console.warn(`Terms directory '${termsDir}' does not exist. Creating an empty term index.`);
+            Logger.warn(`Terms directory '${termsDir}' does not exist. Creating an empty term index.`);
         } else {
             // Get list of files and filter them
             files = fs.readdirSync(termsDir).filter(shouldProcessFile);
         }
         
         if (files.length === 0) {
-            console.log('Warning: No term files found to process.');
+            Logger.warn('No term files found to process.');
         }
         
         const filePaths = files.map(file => specTermDirectoryName[0] + '/' + file);
@@ -132,12 +133,12 @@ function createTermIndex() {
         // Write the term index file
         try {
             fs.writeJsonSync(outputPathJSON, filePaths, { spaces: 2 });
-            console.log(`✅ Term index created with ${files.length} terms. Output: ${outputPathJSON}`);
+            Logger.success(`Term index created with ${files.length} terms. Output: ${outputPathJSON}`);
         } catch (writeError) {
             throw new Error(`Failed to write term index file: ${writeError.message}`);
         }
     } catch (error) {
-        console.error(`❌ Error creating term index: ${error.message}`);
+        Logger.error(`Error creating term index: ${error.message}`);
         throw error;
     }
 }

package/src/{create-versions-index.js → pipeline/configuration/create-versions-index.js}

@@ -14,6 +14,7 @@
 
 const fs = require('fs-extra');
 const path = require('path');
+const Logger = require('../../utils/logger.js');
 
 function createVersionsIndex(outputPath) {
     // Directory containing the version files
@@ -22,7 +23,7 @@ function createVersionsIndex(outputPath) {
     // Check if the directory that holds the versions / snapshots exists, if not create it
     if (!fs.existsSync(versionsDir)) {
         fs.mkdirSync(versionsDir, { recursive: true });
-        console.log('Directory created:', versionsDir);
+        Logger.info('Directory created:', versionsDir);
     }
     
     // Get all directories in the destination directory
@@ -71,9 +72,9 @@ function createVersionsIndex(outputPath) {
     const indexPath = path.join(versionsDir, 'index.html');
     fs.writeFile(indexPath, htmlContent, (err) => {
         if (err) {
-            console.error(`❌ Error writing index file: ${err}`);
+            Logger.error(`Error writing index file: ${err}`);
         } else {
-            console.log(`✅ Index file created at ${indexPath}`);
+            Logger.success(`Index file created at ${indexPath}`);
         }
     });
 }

package/src/{insert-term-index.js → pipeline/configuration/insert-term-index.js}

@@ -9,7 +9,7 @@
  * @function
  * @name insertTermIndex
  * @returns {void}
- * @file src/insert-term-index.js
+ * @file src/pipeline/configuration/insert-term-index.js
  * @author Kor Dwarshuis
  * @version 1.0.0
  * @since 2024-09-02
@@ -26,7 +26,7 @@ function insertTermIndex() {
 
     // Find the index of the "terms-and-definitions-intro.md" file in the markdown paths
     const index = markdownPaths.indexOf('terms-and-definitions-intro.md');
-    
+
     // 
     if (index !== -1) {
         // Insert the items of the "terms" array after the found index

package/src/pipeline/configuration/prepare-spec-configuration.js

@@ -0,0 +1,70 @@
+/**
+ * Handles configuration and initialization tasks before rendering specs.
+ * This module centralizes setup logic to reduce complexity in index.js.
+ * It loads config, runs validators, prepares external specs, and initializes shared variables.
+ */
+
+const fs = require('fs-extra');
+const path = require('path');
+const findPkgDir = require('find-pkg-dir');
+
+const { initialize } = require('../../init.js');
+const Logger = require('../../utils/logger.js');
+const { runJsonKeyValidatorSync } = require('../../json-key-validator.js');
+const { createTermIndex } = require('./create-term-index.js');
+const { insertTermIndex } = require('./insert-term-index.js');
+const { normalizeTerminologyMarkdown } = require('../preprocessing/normalize-terminology-markdown.js');
+
+/**
+ * Initializes configuration and shared variables for spec processing.
+ * @param {Object} options - Options passed to the main function.
+ * @returns {Object} An object containing config, externalSpecsList, template, assets, and other shared variables.
+ */
+async function initializeConfig(options = {}) {
+  try {
+    await initialize();
+
+    runJsonKeyValidatorSync();
+    createTermIndex();
+    insertTermIndex();
+
+    const modulePath = findPkgDir(__dirname);
+    let config = fs.readJsonSync('./.cache/specs-generated.json');
+
+    const createExternalSpecsList = require('./create-external-specs-list.js');
+    const externalSpecsList = createExternalSpecsList(config);
+
+    const createVersionsIndex = require('./create-versions-index.js');
+    createVersionsIndex(config.specs[0].output_path);
+
+    normalizeTerminologyMarkdown(path.join(config.specs[0].spec_directory, config.specs[0].spec_terms_directory));
+
+    let template = fs.readFileSync(path.join(modulePath, 'templates/template.html'), 'utf8');
+    let assets = fs.readJsonSync(modulePath + '/config/asset-map.json');
+
+    // Initialize shared variables
+    let externalReferences;
+    let references = [];
+    let definitions = [];
+    let toc;
+    let specGroups = {};
+    let noticeTitles = {};
+
+    return {
+      config,
+      externalSpecsList,
+      template,
+      assets,
+      externalReferences,
+      references,
+      definitions,
+      specGroups,
+      noticeTitles
+    };
+  } catch (error) {
+    Logger.error(`Error during configuration initialization: ${error.message}`);
+    throw error;
+  }
+}
+
+module.exports = { initializeConfig };

package/src/pipeline/parsing/apply-markdown-it-extensions.js

@@ -0,0 +1,35 @@
+'use strict';
+
+/**
+ * Markdown-it Extensions - Legacy Interface
+ * 
+ * This file provides backward compatibility for the refactored markdown-it extensions.
+ * All the complex logic has been moved to specialized modules in the './markdown-it/' directory
+ * for better maintainability and understanding.
+ * 
+ * The modular approach breaks down the functionality into:
+ * - Table enhancements (Bootstrap styling, responsive wrappers)
+ * - Template-tag syntax processing ([[def:term]], [[tref:spec,term]], etc.)
+ * - Link enhancements (path-based attributes)
+ * - Definition list processing (terminology vs references, term classification)
+ * 
+ * This refactoring reduces cognitive complexity and makes the code more approachable
+ * for developers who are not familiar with the markdown-it library.
+ */
+
+// Import the new modular markdown-it extensions
+const applyMarkdownItExtensions = require('../../markdown-it');
+
+/**
+ * Legacy interface function that maintains compatibility with existing code
+ * 
+ * @param {Object} md - The markdown-it instance to enhance
+ * @param {Array} templates - Array of template-tag handler objects for custom syntax
+ * 
+ * This function simply delegates to the new modular system while maintaining
+ * the same interface that consuming code expects.
+ */
+module.exports = function (md, templates = []) {
+  // Apply all markdown-it enhancements using the new modular system
+  applyMarkdownItExtensions(md, templates);
+};

package/src/pipeline/parsing/create-markdown-parser.js

@@ -0,0 +1,94 @@
+/**
+ * Configures and exports a fully set up markdown-it instance.
+ * This module integrates custom extensions, plugins, and related constants.
+ * It centralizes markdown parsing setup to reduce complexity in index.js.
+ */
+
+const MarkdownIt = require('markdown-it');
+const containers = require('markdown-it-container');
+const path = require('path');
+const fs = require('fs-extra');
+const findPkgDir = require('find-pkg-dir');
+
+const { configurePlugins } = require('../../markdown-it/plugins');
+const { createTemplateTagParser, createSpecParser } = require('../../parsers');
+const { whitespace, templateTags } = require('../../utils/regex-patterns.js');
+
+// Constants used in markdown parsing
+const noticeTypes = {
+  note: 1,
+  issue: 1,
+  example: 1,
+  warning: 1,
+  todo: 1
+};
+// Domain-specific regex patterns for markdown parsing (now centralized)
+const specNameRegex = templateTags.specName;
+const templateTagRegex = templateTags.terminology;
+
+// Load spec corpus
+const modulePath = findPkgDir(__dirname);
+const specCorpus = fs.readJsonSync(path.join(modulePath, 'assets/compiled/refs.json'));
+
+// Global variables (shared across renders)
+let definitions = global.definitions;
+let references = global.references;
+let specGroups = global.specGroups;
+let noticeTitles = global.noticeTitles;
+
+/**
+ * Creates and configures a markdown-it instance with extensions and plugins.
+ * @param {Object} config - Configuration object (e.g., for anchor symbol).
+ * @param {Function} setToc - Function to set the table of contents HTML.
+ * @returns {Object} The configured markdown-it instance.
+ */
+function createMarkdownParser(config, setToc) {
+  // Create parser functions with bound dependencies - cleaner than classes
+  const templateTagParser = createTemplateTagParser(config, global);
+  const specParser = createSpecParser(specCorpus, global);
+
+  let md = MarkdownIt({
+    html: true,
+    linkify: true,
+    typographer: true
+  })
+  .use(require('./apply-markdown-it-extensions.js'), [
+      /*
+        The first extension focuses on template-tag constructs.
+        All complex logic is now delegated to pure functions.
+      */
+      {
+        filter: type => type.match(templateTagRegex),
+        parse: (token, type, primary) => templateTagParser(token, type, primary)
+      },
+      /*
+        The second extension handles specification references.
+        All complex logic is now delegated to pure functions.
+      */
+      {
+        filter: type => type.match(specNameRegex),
+        parse: (token, type, name) => specParser.parseSpecReference(token, type, name),
+        render: (token, type, name) => specParser.renderSpecReference(token, type, name)
+      }
+    ]);
+
+  md = configurePlugins(md, config, containers, noticeTypes, global.noticeTitles, setToc);
+
+  return md;
+}
+
+module.exports = { 
+  createMarkdownParser, 
+  noticeTypes, 
+  spaceRegex: whitespace.oneOrMore, 
+  specNameRegex, 
+  templateTagRegex, 
+  specCorpus, 
+  definitions, 
+  references, 
+  specGroups, 
+  noticeTitles,
+  // Export parsers for direct access if needed
+  createTemplateTagParser,
+  createSpecParser
+};

package/src/pipeline/parsing/create-markdown-parser.test.js

@@ -0,0 +1,49 @@
+/**
+ * Markdown Parser Integration Tests - Functional Style
+ *
+ * These tests verify that the markdown parser integration works correctly
+ * and maintains backward compatibility while providing better testability.
+ */
+
+const {
+  createMarkdownParser
+} = require('./create-markdown-parser.js');
+
+// Tests for integrating the functional parser system with markdown processing
+describe('Markdown Parser Integration', () => {
+  let mockConfig;
+
+  beforeEach(() => {
+    mockConfig = {
+      specs: [{
+        external_specs: [{
+          external_spec: 'test-spec',
+          gh_page: 'https://example.com/spec'
+        }]
+      }]
+    };
+  });
+
+  // Test: Can the system create a markdown parser using the functional approach?
+  test('should create parser with functional system', () => {
+    const mockSetToc = jest.fn();
+    const parser = createMarkdownParser(mockConfig, mockSetToc);
+
+    expect(parser).toBeDefined();
+    expect(parser.render).toBeDefined();
+  });
+
+  // Test: Does the refactored system maintain compatibility with existing markdown processing?
+  test('should maintain backward compatibility', () => {
+    const mockSetToc = jest.fn();
+    const parser = createMarkdownParser(mockConfig, mockSetToc);
+
+    // Test basic markdown rendering still works
+    const basicMarkdown = '# Test Heading\n\nSome content.';
+    const result = parser.render(basicMarkdown);
+
+    expect(result).toContain('<h1');
+    expect(result).toContain('Test Heading');
+    expect(result).toContain('<p>Some content.</p>');
+  });
+});

package/src/{html-dom-processor.js → pipeline/postprocessing/definition-list-postprocessor.js}

@@ -100,7 +100,10 @@ function fixDefinitionListStructure(html) {
   });
 
   // Find any transcluded term dt elements anywhere in the document
-  const transcludedTerms = document.querySelectorAll('dt.transcluded-xref-term');
+  const transcludedTerms = document.querySelectorAll('dt.term-external, dt.term-local');
+  
+  // Also find any tref spans that are in paragraphs (standalone trefs)
+  const standaloneTrefSpans = document.querySelectorAll('p span.term-external');
 
   let mainDl = null;
 
@@ -109,7 +112,7 @@ function fixDefinitionListStructure(html) {
     mainDl = dlElements[0]; // Use the first one
   }
   // If we have transcluded terms but no main dl, we need to create one
-  else if (transcludedTerms.length > 0) {
+  else if (transcludedTerms.length > 0 || standaloneTrefSpans.length > 0) {
     // Create a new dl element with the right class
     mainDl = document.createElement('dl');
     mainDl.className = 'terms-and-definitions-list';
@@ -125,10 +128,18 @@ function fixDefinitionListStructure(html) {
         marker.parentNode.appendChild(mainDl);
       }
     } else {
-      // Fallback to the original approach if marker isn't found
-      const firstTerm = transcludedTerms[0];
-      const insertPoint = firstTerm.parentNode;
-      insertPoint.parentNode.insertBefore(mainDl, insertPoint);
+      // Fallback - insert before the first term we can find (dt or standalone span)
+      let firstTerm = transcludedTerms[0];
+      if (!firstTerm && standaloneTrefSpans.length > 0) {
+        firstTerm = standaloneTrefSpans[0].closest('p');
+      }
+      if (firstTerm) {
+        const insertPoint = firstTerm.parentNode;
+        insertPoint.parentNode.insertBefore(mainDl, insertPoint);
+      } else {
+        // Last resort - append to body
+        document.body.appendChild(mainDl);
+      }
     }
   }
 
@@ -160,6 +171,30 @@ function fixDefinitionListStructure(html) {
     return group;
   }
 
+  // First, process any standalone tref spans in paragraphs and convert them to dt elements
+  standaloneTrefSpans.forEach(trefSpan => {
+    const paragraph = trefSpan.closest('p');
+    if (paragraph) {
+      // Create a new dt element for this tref
+      const newDt = document.createElement('dt');
+      newDt.className = 'term-external';
+      
+      // Move the tref span into the dt
+      newDt.appendChild(trefSpan.cloneNode(true));
+      
+      // Create an empty dd element to satisfy definition list structure
+      const newDd = document.createElement('dd');
+      newDd.innerHTML = ''; // Truly empty, no hacks
+      
+      // Add both to the main dl
+      mainDl.appendChild(newDt);
+      mainDl.appendChild(newDd);
+      
+      // Remove the paragraph
+      paragraph.parentNode.removeChild(paragraph);
+    }
+  });
+
   // Process all transcluded terms and move them with their dd elements
   transcludedTerms.forEach(dt => {
     // Check if this dt is not already inside our main dl
@@ -269,10 +304,34 @@ function fixDefinitionListStructure(html) {
           currentNode.parentNode.removeChild(currentNode);
         }
       }
-      else if (currentNode.tagName === 'P' &&
-        (!currentNode.textContent || currentNode.textContent.trim() === '')) {
-        // Remove empty paragraphs - these break the list structure
-        currentNode.parentNode.removeChild(currentNode);
+      else if (currentNode.tagName === 'P') {
+        // Check if this paragraph contains a standalone tref term
+        const trefSpan = currentNode.querySelector('span.term-external');
+        if (trefSpan) {
+          // Create a new dt element for this tref
+          const newDt = document.createElement('dt');
+          newDt.className = 'term-external';
+          
+          // Move the tref span into the dt
+          newDt.appendChild(trefSpan.cloneNode(true));
+          
+          // Create an empty dd element to satisfy definition list structure
+          const newDd = document.createElement('dd');
+          newDd.innerHTML = ''; // Truly empty, no hacks
+          
+          // Add both to the main dl
+          mainDl.appendChild(newDt);
+          mainDl.appendChild(newDd);
+          
+          // Remove the paragraph
+          currentNode.parentNode.removeChild(currentNode);
+        }
+        else if ((!currentNode.textContent || currentNode.textContent.trim() === '') && 
+                 currentNode.children.length === 0) {
+          // Remove truly empty paragraphs (no text and no child elements) - these break the list structure
+          // This prevents removal of paragraphs containing only images or other elements
+          currentNode.parentNode.removeChild(currentNode);
+        }
       }
     }
 

package/src/{escape-handler.js → pipeline/preprocessing/escape-processor.js}

@@ -8,6 +8,8 @@
  * 3. Post-processing: Restore escaped sequences as literals
  */
 
+const { escaping } = require('../../utils/regex-patterns.js');
+
 const ESCAPED_PLACEHOLDER = '__SPEC_UP_ESCAPED_TAG__';
 
 /**
@@ -40,7 +42,7 @@ function postProcessEscapes(text) {
   }
   
   // Restore placeholders to literal [[
-  return text.replace(new RegExp(ESCAPED_PLACEHOLDER, 'g'), '[[');
+  return text.replace(escaping.placeholderRegex, '[[');
 }
 
 /**