spec-up-t 1.1.54 → 1.2.0

package/src/asset-map.json

@@ -2,23 +2,24 @@
   "head": {
     "css": [
       "assets/css/theme-vars.css",
+      "assets/css/bootstrap.min.css",
       "assets/css/custom-elements.css",
       "assets/css/prism.css",
       "assets/css/chart.css",
+      "assets/css/header-navbar.css",
+      "assets/css/sidebar-toc.css",
+      "assets/css/terms-and-definitions.css",
       "assets/css/edit-term-buttons.css",
       "assets/css/search.css",
       "assets/css/highlightMenuItems.css",
       "assets/css/backToTop.css",
       "assets/css/notyf.css",
       "assets/css/collapse-definitions.css",
+      "assets/css/create-term-filter.css",
       "assets/css/collapse-meta-info.css",
       "assets/css/modal.css",
       "assets/css/create-alphabet-index.css",
       "assets/css/pdf-download.css",
-      "assets/css/terms-and-definitions.css",
-      "assets/css/bootstrap.min.css",
-      "assets/css/header-navbar.css",
-      "assets/css/sidebar-toc.css",
       "assets/css/loader.css",
       "assets/css/external-links.css",
       "assets/css/repo-issues.css",
@@ -56,12 +57,15 @@
       "assets/js/pdf-download.js",
       "assets/js/insert-trefs.js",
       "assets/js/collapse-definitions.js",
+      "assets/js/create-term-filter.js",
       "assets/js/collapse-meta-info.js",
       "assets/js/add-href-to-snapshot-link.js",
       "assets/js/adjust-font-size.js",
+      "assets/js/toggle-dense-info.js",
       "assets/js/close-off-canvas-menu.js",
+
       "assets/js/index.js",
-      "assets/js/css-helper.js",
+      "assets/js/horizontal-scroll-hint.js",
       "assets/js/bootstrap.bundle.min.js"
     ]
   }

package/src/collect-external-references.js

@@ -42,6 +42,8 @@
 
 
 const { shouldProcessFile } = require('./utils/file-filter');
+const path = require('path');
+
 function isXTrefInMarkdown(xtref, markdownContent) {
     const regex = new RegExp(`\\[\\[(?:x|t)ref:${xtref.externalSpec},\\s*${xtref.term}\\]\\]`, 'g');
     return regex.test(markdownContent);
@@ -192,22 +194,26 @@ function collectExternalReferences(options = {}) {
         // Collect all directories that contain files with a term and definition
         // This maps over the specs in the config file and constructs paths to directories
         // where the term definition files are located.
-        const specTermsDirectories = config.specs.map(spec => spec.spec_directory + '/' + spec.spec_terms_directory);
+        const specTermsDirectories = config.specs.map(spec => 
+            path.join(spec.spec_directory, spec.spec_terms_directory)
+        );
 
         // Ensure that the 'output' directory exists, creating it if necessary.
-        if (!fs.existsSync('output')) {
-            fs.mkdirSync('output');
+        const outputDir = 'output';
+        if (!fs.existsSync(outputDir)) {
+            fs.mkdirSync(outputDir);
         }
 
         // Ensure that the 'output/xtrefs-history' directory exists, creating it if necessary.
-        if (!fs.existsSync('output/xtrefs-history')) {
-            fs.mkdirSync('output/xtrefs-history');
+        const xtrefsHistoryDir = path.join(outputDir, 'xtrefs-history');
+        if (!fs.existsSync(xtrefsHistoryDir)) {
+            fs.mkdirSync(xtrefsHistoryDir);
         }
 
         // Define paths for various output files, including JSON and JS files.
-        const outputPathJSON = 'output/xtrefs-data.json';
-        const outputPathJS = 'output/xtrefs-data.js';
-        const outputPathJSTimeStamped = 'output/xtrefs-history/xtrefs-data-' + Date.now() + '.js';
+        const outputPathJSON = path.join(outputDir, 'xtrefs-data.json');
+        const outputPathJS = path.join(outputDir, 'xtrefs-data.js');
+        const outputPathJSTimeStamped = path.join(xtrefsHistoryDir, `xtrefs-data-${Date.now()}.js`);
 
         // Function to extend xtref objects with additional information, such as repository URL and directory information.
         function extendXTrefs(config, xtrefs) {
@@ -280,7 +286,8 @@ function collectExternalReferences(options = {}) {
         specTermsDirectories.forEach(specDirectory => {
             fs.readdirSync(specDirectory).forEach(file => {
                 if (shouldProcessFile(file)) {
-                    const markdown = fs.readFileSync(`${specDirectory}/${file}`, 'utf8');
+                    const filePath = path.join(specDirectory, file);
+                    const markdown = fs.readFileSync(filePath, 'utf8');
                     allMarkdownContent += markdown;
                 }
             });

package/src/collectExternalReferences/fetchTermsFromIndex.js

@@ -0,0 +1,328 @@
+/**
+ * @file fetchTermsFromIndex.js
+ * @description Fetches terms and definitions from external repository's index.html
+ * @author Generated with assistance from GitHub Copilot
+ * @version 1.0.0
+ * @since 2025-04-15
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { JSDOM } = require('jsdom');
+const axios = require('axios');
+const { addPath, getPath, getAllPaths } = require('../config/paths');
+const crypto = require('crypto');
+
+// Directory to store cached files
+const CACHE_DIR = getPath('githubcache');
+
+/**
+ * Generates a cache key based on repository information
+ * @param {string} owner - Repository owner
+ * @param {string} repo - Repository name
+ * @returns {string} - Cache key
+ */
+function generateCacheKey(owner, repo) {
+    const input = `${owner}-${repo}-index`;
+    return crypto.createHash('md5').update(input).digest('hex');
+}
+
+/**
+ * Checks if a cached version exists and is valid
+ * @param {string} cacheKey - Cache key
+ * @param {object} options - Options object
+ * @param {number} options.cacheTTL - Time-to-live for cache in milliseconds (default: 24 hours)
+ * @returns {object|null} - Cached data or null if not found or expired
+ */
+function getFromCache(cacheKey, options = {}) {
+    const cachePath = path.join(CACHE_DIR, `${cacheKey}.json`);
+    // const cacheTTL = options.cacheTTL || 24 * 60 * 60 * 1000; // Default: 24 hours
+    const cacheTTL = 0;
+
+    if (!fs.existsSync(cachePath)) {
+        return null;
+    }
+
+    const cacheData = JSON.parse(fs.readFileSync(cachePath, 'utf8'));
+    const cacheTime = new Date(cacheData.timestamp).getTime();
+    const currentTime = new Date().getTime();
+
+    // Check if cache is expired
+    if (currentTime - cacheTime > cacheTTL) {
+        console.log(`Cache expired for key: ${cacheKey}`);
+        return null;
+    }
+
+    console.log(`Using cached data for key: ${cacheKey}`);
+    return cacheData;
+}
+
+/**
+ * Saves data to cache
+ * @param {string} cacheKey - Cache key
+ * @param {object} data - Data to cache
+ */
+function saveToCache(cacheKey, data) {
+    const cachePath = path.join(CACHE_DIR, `${cacheKey}.json`);
+    const cacheData = {
+        timestamp: new Date().toISOString(),
+        ...data
+    };
+    fs.writeFileSync(cachePath, JSON.stringify(cacheData, null, 2));
+    console.log(`Saved to cache: ${cacheKey}`);
+}
+
+/**
+ * Fetches the latest commit hash for a specific file in a repository
+ * @param {string} token - GitHub API Token
+ * @param {string} owner - Repository owner
+ * @param {string} repo - Repository name
+ * @param {string} filePath - Path to the file in the repository
+ * @param {object} headers - Request headers
+ * @returns {string|null} - Latest commit hash or null if error
+ */
+async function getFileCommitHash(token, owner, repo, filePath, headers) {
+    try {
+        // Normalize the file path to ensure it doesn't have leading slash
+        const normalizedPath = filePath.replace(/^\//, '');
+        
+        // Construct API URL to get commits for the specific file
+        const commitsUrl = `https://api.github.com/repos/${owner}/${repo}/commits?path=${normalizedPath}&per_page=1`;
+        console.log(`Fetching latest commit for file: ${commitsUrl}`);
+        
+        const response = await axios.get(commitsUrl, { headers });
+        
+        if (response.status !== 200 || !response.data || response.data.length === 0) {
+            console.log(`❌ Could not find commit information for ${filePath}`);
+            return null;
+        }
+        
+        // Return the SHA of the latest commit
+        return response.data[0].sha;
+    } catch (error) {
+        console.error(`❌ Error fetching commit hash: ${error.message}`);
+        return null;
+    }
+}
+
+/**
+ * Fetches all terms and definitions from a repository's index.html
+ * @param {string} token - GitHub API Token
+ * @param {string} owner - Repository owner
+ * @param {string} repo - Repository name 
+ * @param {object} options - Additional options
+ * @returns {object|null} - Object containing all terms or null if error
+ */
+async function fetchAllTermsFromIndex(token, owner, repo, options = {}) {
+    try {
+        // Generate cache key based on repo information
+        const cacheKey = generateCacheKey(owner, repo);
+        let cachedData = null;
+
+        // Check cache first if caching is enabled
+        if (options.cache !== false) {
+            cachedData = getFromCache(cacheKey, options);
+            if (cachedData) {
+                return cachedData;
+            }
+        }
+
+        // Configure headers for GitHub API
+        const headers = {};
+        if (token) {
+            headers['Authorization'] = `token ${token}`;
+        }
+
+        // Get the specs.json content from the repository to find the output_path
+        const specsJsonUrl = `https://api.github.com/repos/${owner}/${repo}/contents/specs.json`;
+        console.log(`Fetching specs.json from: ${specsJsonUrl}`);
+
+        // Fetch specs.json content
+        const specsJsonResponse = await axios.get(specsJsonUrl, { headers });
+        if (specsJsonResponse.status !== 200) {
+            console.log(`❌ Could not find specs.json in repository ${owner}/${repo}`);
+            return null;
+        }
+
+        // Decode specs.json content from base64
+        const specsJsonContent = Buffer.from(specsJsonResponse.data.content, 'base64').toString('utf8');
+        const specsJson = JSON.parse(specsJsonContent);
+        
+        // Get the output_path from specs.json
+        const outputPath = specsJson.specs[0].output_path;
+        if (!outputPath) {
+            console.log(`❌ No output_path found in specs.json for repository ${owner}/${repo}`);
+            return null;
+        }
+
+        // Fix: Properly normalize the output path to ensure it doesn't have leading "./" or trailing "/"
+        const normalizedOutputPath = outputPath.replace(/^\.\//, '').replace(/\/$/, '');
+        
+        // Create the path to the index.html file
+        const indexHtmlPath = `${normalizedOutputPath}/index.html`;
+        
+        // Fetch the index.html content with properly constructed URL
+        const indexHtmlUrl = `https://raw.githubusercontent.com/${owner}/${repo}/main/${indexHtmlPath}`;
+        console.log(`Fetching index.html from: ${indexHtmlUrl}`);
+        
+        const indexHtmlResponse = await axios.get(indexHtmlUrl, { headers });
+        if (indexHtmlResponse.status !== 200) {
+            console.log(`❌ Could not find index.html at ${indexHtmlUrl}`);
+            return null;
+        }
+
+        // Get the commit hash for the index.html file
+        const commitHash = await getFileCommitHash(token, owner, repo, indexHtmlPath, headers);
+        if (!commitHash) {
+            console.log(`⚠️ Could not get commit hash for index.html, continuing without it`);
+        }
+
+        const htmlContent = indexHtmlResponse.data;
+        
+        // Parse HTML using JSDOM
+        const dom = new JSDOM(htmlContent);
+        const document = dom.window.document;
+        
+        // Find all term definition lists with class "terms-and-definitions-list"
+        const termDlList = document.querySelector('dl.terms-and-definitions-list');
+        if (!termDlList) {
+            console.log(`❌ No terms-and-definitions-list found in ${indexHtmlUrl}`);
+            return null;
+        }
+
+        // Extract all terms and definitions
+        const terms = [];
+        let dtElements = termDlList.querySelectorAll('dt');
+        
+        dtElements.forEach(dt => {
+            // Find the term span that starts with id="term:
+            const termSpan = dt.querySelector('span[id^="term:"]');
+            if (!termSpan) return;
+
+            // Get the term text (all text content, excluding nested spans)
+            let termText = '';
+            for (let node of termSpan.childNodes) {
+                if (node.nodeType === dom.window.Node.TEXT_NODE) {
+                    termText += node.textContent.trim();
+                }
+            }
+            
+            // If no text found, try to get the full text content
+            if (!termText) {
+                termText = termSpan.textContent.trim();
+            }
+            
+            // Skip empty terms
+            if (!termText) return;
+            
+            // Find all corresponding definition elements
+            let dds = [];
+            let nextElement = dt.nextElementSibling;
+            
+            // Collect all consecutive <dd> elements until we reach another <dt>
+            while (nextElement && nextElement.tagName.toLowerCase() === 'dd') {
+                dds.push(nextElement.outerHTML);
+                nextElement = nextElement.nextElementSibling;
+            }
+            
+            terms.push({
+                term: termText,
+                definition: dds.join('\n')
+            });
+        });
+
+        // Store all terms in a JSON file with timestamp
+        const timestamp = Date.now();
+        const outputDir = path.join(CACHE_DIR);
+        if (!fs.existsSync(outputDir)) {
+            fs.mkdirSync(outputDir, { recursive: true });
+        }
+        
+        // Create output filename with timestamp
+        const outputFileName = `${timestamp}-${owner}-${repo}-terms.json`;
+        const outputFilePath = path.join(outputDir, outputFileName);
+        
+        // Create the result object
+        const result = {
+            timestamp,
+            repository: `${owner}/${repo}`,
+            terms,
+            sha: commitHash, // Use the commit hash of the index.html file
+            avatarUrl: null,
+            outputFileName
+        };
+        
+        // Save all terms to file
+        fs.writeFileSync(outputFilePath, JSON.stringify(result, null, 2));
+        console.log(`✅ Saved ${terms.length} terms to ${outputFilePath}`);
+        
+        // Save to cache if enabled
+        if (options.cache !== false) {
+            saveToCache(cacheKey, result);
+        }
+        
+        return result;
+
+    } catch (error) {
+        if (error.response) {
+            if (error.response.status === 404) {
+                console.log(`❌ Resource not found: ${error.config.url}`);
+            } else if (error.response.status === 403 && error.response.headers['x-ratelimit-remaining'] === '0') {
+                const resetTime = new Date(parseInt(error.response.headers['x-ratelimit-reset']) * 1000);
+                console.error(`❌ GitHub API rate limit exceeded. Try again after ${resetTime.toLocaleString()}`);
+            } else {
+                console.error(`❌ Error fetching data: ${error.response.status} ${error.response.statusText}`);
+            }
+        } else {
+            console.error(`❌ Error fetching term: ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Fetches a specific term from repository's index.html
+ * This is a wrapper that uses fetchAllTermsFromIndex for efficiency
+ * @param {string} token - GitHub API Token
+ * @param {string} term - The specific term to look for
+ * @param {string} owner - Repository owner
+ * @param {string} repo - Repository name
+ * @param {string} termsDir - Directory containing term definitions (not used in this implementation)
+ * @param {object} options - Additional options
+ * @returns {object|null} - Found term data or null if not found
+ */
+async function fetchTermsFromIndex(token, term, owner, repo, termsDir, options = {}) {
+    // First get all terms from the repository (which is cached)
+    const allTermsData = await fetchAllTermsFromIndex(token, owner, repo, options);
+    
+    if (!allTermsData || !allTermsData.terms) {
+        return null;
+    }
+    
+    // Find the specific term
+    const foundTerm = allTermsData.terms.find(t => t.term.toLowerCase() === term.toLowerCase());
+    
+    if (foundTerm) {
+        console.log(`Found term '${term}' in repository ${owner}/${repo}`);
+        return {
+            term: foundTerm.term,
+            content: foundTerm.definition,
+            sha: allTermsData.sha,
+            repository: {
+                owner: {
+                    login: owner,
+                    avatar_url: allTermsData.avatarUrl
+                },
+                name: repo
+            }
+        };
+    } else {
+        console.log(`❌ Term "${term}" not found in repository ${owner}/${repo}`);
+        return null;
+    }
+}
+
+module.exports = {
+    fetchTermsFromIndex,
+    fetchAllTermsFromIndex  // Export the function to fetch all terms as well
+};

package/src/collectExternalReferences/processXTrefsData.js

@@ -1,14 +1,14 @@
 const fs = require('fs');
-const { fetchTermsFromGitHubRepository } = require('./fetchTermsFromGitHubRepository.js');
+const { fetchTermsFromIndex, fetchAllTermsFromIndex } = require('./fetchTermsFromIndex.js');
 const { matchTerm } = require('./matchTerm.js');
 const { addPath, getPath, getAllPaths } = require('../config/paths');
+const path = require('path');
 
 // Directory to store cached files
 const CACHE_DIR = getPath('githubcache');
 
 async function processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, outputPathJS, outputPathJSTimeStamped, options) {
     try {
-
         // Clear the cache (remove the cache directory) if the cache option is set to false
         if (options.cache === false) {
             if (fs.existsSync(CACHE_DIR)) {
@@ -20,24 +20,78 @@ async function processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, ou
         if (!fs.existsSync(CACHE_DIR)) {
             fs.mkdirSync(CACHE_DIR, { recursive: true });
         }
+        
+        // Filter out incomplete xtrefs that don't have proper repository information
+        allXTrefs.xtrefs = allXTrefs.xtrefs.filter(xtref => {
+            if (!xtref.owner || !xtref.repo || !xtref.repoUrl) {
+                console.log(`⚠️ Removing incomplete reference: ${xtref.externalSpec}, ${xtref.term}`);
+                return false;
+            }
+            return true;
+        });
+        
+        // Group xtrefs by repository to avoid multiple downloads of the same index.html
+        const xrefsByRepo = allXTrefs.xtrefs.reduce((groups, xtref) => {
+            const repoKey = `${xtref.owner}/${xtref.repo}`;
+            if (!groups[repoKey]) {
+                groups[repoKey] = {
+                    owner: xtref.owner,
+                    repo: xtref.repo,
+                    xtrefs: []
+                };
+            }
+            groups[repoKey].xtrefs.push(xtref);
+            return groups;
+        }, {});
 
-        for (let xtref of allXTrefs.xtrefs) {
-            // Go and look if the term is in the external repository and if so, get the commit hash, and other meta info plus the content of the file
-            const item = await fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, xtref.term, xtref.owner, xtref.repo, xtref.terms_dir, options);
-
-            // // Check if fetchedData.data is defined
-            if (item !== null && matchTerm(item.content, xtref.term)) {
-                xtref.commitHash = item.sha;
-                xtref.content = item.content;
-                xtref.avatarUrl = item.repository.owner.avatar_url;
-                console.log(`✅ Match found for term: ${xtref.term} in ${xtref.externalSpec};`);
-                console.log("============================================\n\n");
-            } else {
-                xtref.commitHash = "not found";
-                xtref.content = "This term was not found in the external repository.";
-                console.log(`ℹ️ No match found for term: ${xtref.term} in ${xtref.externalSpec};`);
-                console.log("============================================\n\n");
+        console.log(`✅ Grouped ${allXTrefs.xtrefs.length} terms into ${Object.keys(xrefsByRepo).length} repositories`);
+        
+        // Process each repository once
+        for (const repoKey of Object.keys(xrefsByRepo)) {
+            const repoGroup = xrefsByRepo[repoKey];
+            console.log(`Processing repository: ${repoKey} (${repoGroup.xtrefs.length} terms)`);
+            
+            // First, fetch all terms from this repository
+            const allTermsData = await fetchAllTermsFromIndex(
+                GITHUB_API_TOKEN, 
+                repoGroup.owner, 
+                repoGroup.repo, 
+                options
+            );
+            
+            if (!allTermsData) {
+                console.log(`❌ Could not fetch terms from repository ${repoKey}`);
+                // Mark all terms from this repo as not found
+                repoGroup.xtrefs.forEach(xtref => {
+                    xtref.commitHash = "not found";
+                    xtref.content = "This term was not found in the external repository.";
+                    xtref.avatarUrl = null;
+                });
+                continue; // Skip to next repository
+            }
+            
+            // Now process each term in this repository
+            for (const xtref of repoGroup.xtrefs) {
+                // Find the term in the pre-fetched data
+                const foundTerm = allTermsData.terms.find(
+                    t => t.term.toLowerCase() === xtref.term.toLowerCase()
+                );
+                
+                if (foundTerm) {
+                    xtref.commitHash = allTermsData.sha;
+                    xtref.content = foundTerm.definition;
+                    xtref.avatarUrl = allTermsData.avatarUrl;
+                    console.log(`✅ Match found for term: ${xtref.term} in ${xtref.externalSpec}`);
+                } else {
+                    xtref.commitHash = "not found";
+                    xtref.content = "This term was not found in the external repository.";
+                    xtref.avatarUrl = null;
+                    console.log(`ℹ️ No match found for term: ${xtref.term} in ${xtref.externalSpec}`);
+                }
             }
+            
+            console.log(`✅ Finished processing repository: ${repoKey}`);
+            console.log("============================================\n\n");
         }
 
         const allXTrefsStr = JSON.stringify(allXTrefs, null, 2);
@@ -46,6 +100,7 @@ async function processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, ou
         fs.writeFileSync(outputPathJS, stringReadyForFileWrite, 'utf8');
         fs.writeFileSync(outputPathJSTimeStamped, stringReadyForFileWrite, 'utf8');
 
+        // This will run index.js
         require('../../index.js')({ nowatch: true });
     } catch (error) {
         console.error("An error occurred:", error);