spec-up-t 1.2.9 → 1.3.0

package/src/collect-external-references.test.js

@@ -93,6 +93,46 @@ And here we reference it again using [[tref:kmg-1,authentic-chained-data-contain
 ### Conclusion
 That's all about these references.`,
             shouldMatch: true
+        },
+
+        // Test cases for aliases - the function should match when the original term exists regardless of alias
+        {
+            name: 'tref with alias should match based on original term',
+            xtref: { externalSpec: 'vlei1', term: 'vlei-ecosystem-governance-framework' },
+            markdown: '[[tref:vlei1, vlei-ecosystem-governance-framework, vEGF]]',
+            shouldMatch: true
+        },
+        {
+            name: 'xref with alias should match based on original term',
+            xtref: { externalSpec: 'vlei1', term: 'vlei-ecosystem-governance-framework' },
+            markdown: '[[xref:vlei1, vlei-ecosystem-governance-framework, vEGF]]',
+            shouldMatch: true
+        },
+        {
+            name: 'multiple aliases for same term should match',
+            xtref: { externalSpec: 'spec1', term: 'long-term-name' },
+            markdown: 'Text [[tref:spec1, long-term-name, alias1]] and [[tref:spec1, long-term-name, alias2]]',
+            shouldMatch: true
+        },
+        {
+            name: 'tref with spaces in alias should match',
+            xtref: { externalSpec: 'spec1', term: 'term1' },
+            markdown: '[[tref:spec1, term1, alias with spaces]]',
+            shouldMatch: true
+        },
+
+        // Test case for the specific issue with hyphens and spaces
+        {
+            name: 'external spec and term with hyphens and alias should match',
+            xtref: { externalSpec: 'vlei-glossary', term: 'vlei-ecosystem-governance-framework' },
+            markdown: '[[tref: vlei-glossary, vlei-ecosystem-governance-framework, vegf]]',
+            shouldMatch: true
+        },
+        {
+            name: 'external spec and term with hyphens without alias should match',
+            xtref: { externalSpec: 'vlei-glossary', term: 'vlei-ecosystem-governance-framework' },
+            markdown: '[[tref: vlei-glossary, vlei-ecosystem-governance-framework]]',
+            shouldMatch: true
         }
     ];
 
@@ -119,12 +159,15 @@ describe('addNewXTrefsFromMarkdown', () => {
         });
     });
 
-    it('should not add duplicate xtrefs', () => {
-        const markdownContent = "Content [[xref:specA, termA]] and again [[xref:specA, termA]]";
+    it('should not add duplicate xtrefs with same spec and term but different aliases', () => {
+        const markdownContent = "Content [[xref:specA, termA]] and again [[xref:specA, termA, aliasA]]";
         const allXTrefs = { xtrefs: [] };
         const updatedXTrefs = addNewXTrefsFromMarkdown(markdownContent, allXTrefs);
 
         expect(updatedXTrefs.xtrefs.length).toBe(1);
+        expect(updatedXTrefs.xtrefs[0].term).toBe('termA');
+        expect(updatedXTrefs.xtrefs[0].externalSpec).toBe('specA');
+        // The first one found will be used (without alias in this case)
     });
 
     it('should add multiple distinct xtrefs', () => {
@@ -149,4 +192,112 @@ describe('addNewXTrefsFromMarkdown', () => {
         expect(updatedXTrefs.xtrefs.length).toBe(0);
     });
 
+    it('should add a new tref with alias from markdown content', () => {
+        const markdownContent = "Some text [[tref:specA, termA, aliasA]] more text";
+        const allXTrefs = { xtrefs: [] };
+        const updatedXTrefs = addNewXTrefsFromMarkdown(markdownContent, allXTrefs);
+
+        expect(updatedXTrefs.xtrefs.length).toBe(1);
+        expect(updatedXTrefs.xtrefs[0]).toEqual({
+            externalSpec: 'specA',
+            term: 'termA',
+            alias: 'aliasA'
+        });
+    });
+
+    it('should add a new xref with alias from markdown content', () => {
+        const markdownContent = "Some text [[xref:specA, termA, aliasA]] more text";
+        const allXTrefs = { xtrefs: [] };
+        const updatedXTrefs = addNewXTrefsFromMarkdown(markdownContent, allXTrefs);
+
+        expect(updatedXTrefs.xtrefs.length).toBe(1);
+        expect(updatedXTrefs.xtrefs[0]).toEqual({
+            externalSpec: 'specA',
+            term: 'termA',
+            alias: 'aliasA'
+        });
+    });
+
+    it('should handle tref without alias (backwards compatibility)', () => {
+        const markdownContent = "Some text [[tref:specA, termA]] more text";
+        const allXTrefs = { xtrefs: [] };
+        const updatedXTrefs = addNewXTrefsFromMarkdown(markdownContent, allXTrefs);
+
+        expect(updatedXTrefs.xtrefs.length).toBe(1);
+        expect(updatedXTrefs.xtrefs[0]).toEqual({
+            externalSpec: 'specA',
+            term: 'termA'
+        });
+        expect(updatedXTrefs.xtrefs[0].alias).toBeUndefined();
+    });
+
+});
+
+
+describe('processXTref', () => {
+    const processXTref = require('./collect-external-references').processXTref;
+
+    it('should process basic xref without alias', () => {
+        const xtref = '[[xref:specA,termA]]';
+        const result = processXTref(xtref);
+        
+        expect(result).toEqual({
+            externalSpec: 'specA',
+            term: 'termA'
+        });
+    });
+
+    it('should process basic tref without alias', () => {
+        const xtref = '[[tref:specA,termA]]';
+        const result = processXTref(xtref);
+        
+        expect(result).toEqual({
+            externalSpec: 'specA',
+            term: 'termA'
+        });
+    });
+
+    it('should process tref with alias', () => {
+        const xtref = '[[tref:specA,termA,aliasA]]';
+        const result = processXTref(xtref);
+        
+        expect(result).toEqual({
+            externalSpec: 'specA',
+            term: 'termA',
+            alias: 'aliasA'
+        });
+    });
+
+    it('should process xref with alias', () => {
+        const xtref = '[[xref:specA,termA,aliasA]]';
+        const result = processXTref(xtref);
+        
+        expect(result).toEqual({
+            externalSpec: 'specA',
+            term: 'termA',
+            alias: 'aliasA'
+        });
+    });
+
+    it('should handle spaces in parameters', () => {
+        const xtref = '[[tref: specA , termA , aliasA ]]';
+        const result = processXTref(xtref);
+        
+        expect(result).toEqual({
+            externalSpec: 'specA',
+            term: 'termA',
+            alias: 'aliasA'
+        });
+    });
+
+    it('should ignore empty alias parameter', () => {
+        const xtref = '[[tref:specA,termA,]]';
+        const result = processXTref(xtref);
+        
+        expect(result).toEqual({
+            externalSpec: 'specA',
+            term: 'termA'
+        });
+        expect(result.alias).toBeUndefined();
+    });
 });

package/src/collectExternalReferences/fetchTermsFromIndex.js

@@ -11,67 +11,11 @@ const path = require('path');
 const { JSDOM } = require('jsdom');
 const axios = require('axios');
 const { addPath, getPath, getAllPaths } = require('../../config/paths');
-const crypto = require('crypto');
 
-// Directory to store cached files
+// Directory to store fetched data files
 const CACHE_DIR = getPath('githubcache');
 
-/**
- * Generates a cache key based on repository information
- * @param {string} owner - Repository owner
- * @param {string} repo - Repository name
- * @returns {string} - Cache key
- */
-function generateCacheKey(owner, repo) {
-    const input = `${owner}-${repo}-index`;
-    return crypto.createHash('md5').update(input).digest('hex');
-}
 
-/**
- * Checks if a cached version exists and is valid
- * @param {string} cacheKey - Cache key
- * @param {object} options - Options object
- * @param {number} options.cacheTTL - Time-to-live for cache in milliseconds (default: 24 hours)
- * @returns {object|null} - Cached data or null if not found or expired
- * @example
- * const cacheTTL = options.cacheTTL || 24 * 60 * 60 * 1000; // Default: 24 hours
- */
-function getFromCache(cacheKey, options = {}) {
-    const cachePath = path.join(CACHE_DIR, `${cacheKey}.json`);
-    const cacheTTL = 0;
-
-    if (!fs.existsSync(cachePath)) {
-        return null;
-    }
-
-    const cacheData = JSON.parse(fs.readFileSync(cachePath, 'utf8'));
-    const cacheTime = new Date(cacheData.timestamp).getTime();
-    const currentTime = new Date().getTime();
-
-    // Check if cache is expired
-    if (currentTime - cacheTime > cacheTTL) {
-        console.log(`Cache expired for key: ${cacheKey}`);
-        return null;
-    }
-
-    console.log(`Using cached data for key: ${cacheKey}`);
-    return cacheData;
-}
-
-/**
- * Saves data to cache
- * @param {string} cacheKey - Cache key
- * @param {object} data - Data to cache
- */
-function saveToCache(cacheKey, data) {
-    const cachePath = path.join(CACHE_DIR, `${cacheKey}.json`);
-    const cacheData = {
-        timestamp: new Date().toISOString(),
-        ...data
-    };
-    fs.writeFileSync(cachePath, JSON.stringify(cacheData, null, 2));
-    console.log(`Saved to cache: ${cacheKey}`);
-}
 
 /**
  * Fetches the latest commit hash for a specific file in a repository
@@ -107,77 +51,93 @@ async function getFileCommitHash(token, owner, repo, filePath, headers) {
 }
 
 /**
- * Fetches all terms and definitions from a repository's index.html
+ * Fetches all terms and definitions from a repository's GitHub Pages index.html
  * @param {string} token - GitHub API Token
  * @param {string} owner - Repository owner
  * @param {string} repo - Repository name 
- * @param {object} options - Additional options
+ * @param {object} options - Additional options including ghPageUrl
  * @returns {object|null} - Object containing all terms or null if error
  */
 async function fetchAllTermsFromIndex(token, owner, repo, options = {}) {
     try {
-        // Generate cache key based on repo information
-        const cacheKey = generateCacheKey(owner, repo);
-        let cachedData = null;
-
-        // Check cache first if caching is enabled
-        if (options.cache !== false) {
-            cachedData = getFromCache(cacheKey, options);
-            if (cachedData) {
-                return cachedData;
-            }
-        }
-
         // Configure headers for GitHub API
         const headers = {};
         if (token) {
             headers['Authorization'] = `token ${token}`;
         }
 
-        // Get the specs.json content from the repository to find the output_path
-        const specsJsonUrl = `https://api.github.com/repos/${owner}/${repo}/contents/specs.json`;
-        console.log(`Fetching specs.json from: ${specsJsonUrl}`);
+        // Use GitHub Pages URL if provided in options, otherwise fallback to raw repository
+        let indexHtmlUrl;
+        let commitHash = null;
 
-        // Fetch specs.json content
-        const specsJsonResponse = await axios.get(specsJsonUrl, { headers });
-        if (specsJsonResponse.status !== 200) {
-            console.log(`❌ Could not find specs.json in repository ${owner}/${repo}`);
-            return null;
-        }
+        if (options.ghPageUrl) {
+            // Fetch from GitHub Pages (deployed HTML)
+            indexHtmlUrl = options.ghPageUrl.endsWith('/') ? 
+                `${options.ghPageUrl}index.html` : 
+                `${options.ghPageUrl}/index.html`;
+            console.log(`Fetching index.html from GitHub Pages: ${indexHtmlUrl}`);
+            
+            // For GitHub Pages, we'll try to get the commit hash from the main branch
+            try {
+                const mainBranchUrl = `https://api.github.com/repos/${owner}/${repo}/branches/main`;
+                const branchResponse = await axios.get(mainBranchUrl, { headers });
+                if (branchResponse.status === 200) {
+                    commitHash = branchResponse.data.commit.sha;
+                    console.log(`✅ Got commit hash from main branch: ${commitHash}`);
+                }
+            } catch (error) {
+                console.log(`⚠️ Could not get commit hash from main branch: ${error.message}`);
+            }
+        } else {
+            // Fallback to raw repository method
+            console.log(`⚠️ No GitHub Pages URL provided, falling back to repository method`);
+            
+            // Get the specs.json content from the repository to find the output_path
+            const specsJsonUrl = `https://api.github.com/repos/${owner}/${repo}/contents/specs.json`;
+            console.log(`Fetching specs.json from: ${specsJsonUrl}`);
 
-        // Decode specs.json content from base64
-        const specsJsonContent = Buffer.from(specsJsonResponse.data.content, 'base64').toString('utf8');
-        const specsJson = JSON.parse(specsJsonContent);
-        
-        // Get the output_path from specs.json
-        const outputPath = specsJson.specs[0].output_path;
-        if (!outputPath) {
-            console.log(`❌ No output_path found in specs.json for repository ${owner}/${repo}`);
-            return null;
+            // Fetch specs.json content
+            const specsJsonResponse = await axios.get(specsJsonUrl, { headers });
+            if (specsJsonResponse.status !== 200) {
+                console.log(`❌ Could not find specs.json in repository ${owner}/${repo}`);
+                return null;
+            }
+
+            // Decode specs.json content from base64
+            const specsJsonContent = Buffer.from(specsJsonResponse.data.content, 'base64').toString('utf8');
+            const specsJson = JSON.parse(specsJsonContent);
+            
+            // Get the output_path from specs.json
+            const outputPath = specsJson.specs[0].output_path;
+            if (!outputPath) {
+                console.log(`❌ No output_path found in specs.json for repository ${owner}/${repo}`);
+                return null;
+            }
+
+            // Fix: Properly normalize the output path to ensure it doesn't have leading "./" or trailing "/"
+            const normalizedOutputPath = outputPath.replace(/^\.\//, '').replace(/\/$/, '');
+            
+            // Create the path to the index.html file
+            const indexHtmlPath = `${normalizedOutputPath}/index.html`;
+            
+            // Fetch the index.html content with properly constructed URL
+            indexHtmlUrl = `https://raw.githubusercontent.com/${owner}/${repo}/main/${indexHtmlPath}`;
+            console.log(`Fetching index.html from raw repository: ${indexHtmlUrl}`);
+
+            // Get the commit hash for the index.html file
+            commitHash = await getFileCommitHash(token, owner, repo, indexHtmlPath, headers);
+            if (!commitHash) {
+                console.log(`⚠️ Could not get commit hash for index.html, continuing without it`);
+            }
         }
 
-        // Fix: Properly normalize the output path to ensure it doesn't have leading "./" or trailing "/"
-        const normalizedOutputPath = outputPath.replace(/^\.\//, '').replace(/\/$/, '');
-        
-        // Create the path to the index.html file
-        const indexHtmlPath = `${normalizedOutputPath}/index.html`;
-        
-        // Fetch the index.html content with properly constructed URL
-        const indexHtmlUrl = `https://raw.githubusercontent.com/${owner}/${repo}/main/${indexHtmlPath}`;
-        console.log(`Fetching index.html from: ${indexHtmlUrl}`);
-        
+        // Fetch the index.html content
         const indexHtmlResponse = await axios.get(indexHtmlUrl, { headers });
         if (indexHtmlResponse.status !== 200) {
             console.log(`❌ Could not find index.html at ${indexHtmlUrl}`);
             return null;
         }
 
-        // Get the commit hash for the index.html file
-        const commitHash = await getFileCommitHash(token, owner, repo, indexHtmlPath, headers);
-        if (!commitHash) {
-            console.log(`⚠️ Could not get commit hash for index.html, continuing without it`);
-        }
-
         const htmlContent = indexHtmlResponse.data;
         
         // Parse HTML using JSDOM
@@ -257,11 +217,6 @@ async function fetchAllTermsFromIndex(token, owner, repo, options = {}) {
         fs.writeFileSync(outputFilePath, JSON.stringify(result, null, 2));
         console.log(`✅ Saved ${terms.length} terms to ${outputFilePath}`);
         
-        // Save to cache if enabled
-        if (options.cache !== false) {
-            saveToCache(cacheKey, result);
-        }
-        
         return result;
 
     } catch (error) {

package/src/collectExternalReferences/processXTrefsData.js

@@ -4,19 +4,12 @@ const { matchTerm } = require('./matchTerm.js');
 const { addPath, getPath, getAllPaths } = require('../../config/paths');
 const path = require('path');
 
-// Directory to store cached files
+// Directory to store fetched data files
 const CACHE_DIR = getPath('githubcache');
 
-async function processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, outputPathJS, outputPathJSTimeStamped, options) {
+async function processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, outputPathJS, outputPathJSTimeStamped) {
     try {
-        // Clear the cache (remove the cache directory) if the cache option is set to false
-        if (options.cache === false) {
-            if (fs.existsSync(CACHE_DIR)) {
-                fs.rmdirSync(CACHE_DIR, { recursive: true });
-            }
-        }
-        
-        // Ensure the cache directory exists, so that we can store the fetched data
+        // Ensure the directory exists, so that we can store the fetched data
         if (!fs.existsSync(CACHE_DIR)) {
             fs.mkdirSync(CACHE_DIR, { recursive: true });
         }
@@ -51,12 +44,17 @@ async function processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, ou
             const repoGroup = xrefsByRepo[repoKey];
             console.log(`Processing repository: ${repoKey} (${repoGroup.xtrefs.length} terms)`);
             
+            // Get the GitHub Pages URL from the first xtref in this repo group
+            const ghPageUrl = repoGroup.xtrefs[0]?.ghPageUrl;
+            
             // First, fetch all terms from this repository
             const allTermsData = await fetchAllTermsFromIndex(
                 GITHUB_API_TOKEN, 
                 repoGroup.owner, 
                 repoGroup.repo, 
-                options
+                { 
+                    ghPageUrl: ghPageUrl // Pass the GitHub Pages URL
+                }
             );
             
             if (!allTermsData) {