spec-up-t 1.2.9 → 1.3.0-beta

package/src/collectExternalReferences/fetchTermsFromIndex.js

@@ -11,67 +11,11 @@ const path = require('path');
 const { JSDOM } = require('jsdom');
 const axios = require('axios');
 const { addPath, getPath, getAllPaths } = require('../../config/paths');
-const crypto = require('crypto');
 
-// Directory to store cached files
+// Directory to store fetched data files
 const CACHE_DIR = getPath('githubcache');
 
-/**
- * Generates a cache key based on repository information
- * @param {string} owner - Repository owner
- * @param {string} repo - Repository name
- * @returns {string} - Cache key
- */
-function generateCacheKey(owner, repo) {
-    const input = `${owner}-${repo}-index`;
-    return crypto.createHash('md5').update(input).digest('hex');
-}
 
-/**
- * Checks if a cached version exists and is valid
- * @param {string} cacheKey - Cache key
- * @param {object} options - Options object
- * @param {number} options.cacheTTL - Time-to-live for cache in milliseconds (default: 24 hours)
- * @returns {object|null} - Cached data or null if not found or expired
- * @example
- * const cacheTTL = options.cacheTTL || 24 * 60 * 60 * 1000; // Default: 24 hours
- */
-function getFromCache(cacheKey, options = {}) {
-    const cachePath = path.join(CACHE_DIR, `${cacheKey}.json`);
-    const cacheTTL = 0;
-
-    if (!fs.existsSync(cachePath)) {
-        return null;
-    }
-
-    const cacheData = JSON.parse(fs.readFileSync(cachePath, 'utf8'));
-    const cacheTime = new Date(cacheData.timestamp).getTime();
-    const currentTime = new Date().getTime();
-
-    // Check if cache is expired
-    if (currentTime - cacheTime > cacheTTL) {
-        console.log(`Cache expired for key: ${cacheKey}`);
-        return null;
-    }
-
-    console.log(`Using cached data for key: ${cacheKey}`);
-    return cacheData;
-}
-
-/**
- * Saves data to cache
- * @param {string} cacheKey - Cache key
- * @param {object} data - Data to cache
- */
-function saveToCache(cacheKey, data) {
-    const cachePath = path.join(CACHE_DIR, `${cacheKey}.json`);
-    const cacheData = {
-        timestamp: new Date().toISOString(),
-        ...data
-    };
-    fs.writeFileSync(cachePath, JSON.stringify(cacheData, null, 2));
-    console.log(`Saved to cache: ${cacheKey}`);
-}
 
 /**
  * Fetches the latest commit hash for a specific file in a repository
@@ -107,77 +51,93 @@ async function getFileCommitHash(token, owner, repo, filePath, headers) {
 }
 
 /**
- * Fetches all terms and definitions from a repository's index.html
+ * Fetches all terms and definitions from a repository's GitHub Pages index.html
  * @param {string} token - GitHub API Token
  * @param {string} owner - Repository owner
  * @param {string} repo - Repository name 
- * @param {object} options - Additional options
+ * @param {object} options - Additional options including ghPageUrl
  * @returns {object|null} - Object containing all terms or null if error
  */
 async function fetchAllTermsFromIndex(token, owner, repo, options = {}) {
     try {
-        // Generate cache key based on repo information
-        const cacheKey = generateCacheKey(owner, repo);
-        let cachedData = null;
-
-        // Check cache first if caching is enabled
-        if (options.cache !== false) {
-            cachedData = getFromCache(cacheKey, options);
-            if (cachedData) {
-                return cachedData;
-            }
-        }
-
         // Configure headers for GitHub API
         const headers = {};
         if (token) {
             headers['Authorization'] = `token ${token}`;
         }
 
-        // Get the specs.json content from the repository to find the output_path
-        const specsJsonUrl = `https://api.github.com/repos/${owner}/${repo}/contents/specs.json`;
-        console.log(`Fetching specs.json from: ${specsJsonUrl}`);
+        // Use GitHub Pages URL if provided in options, otherwise fallback to raw repository
+        let indexHtmlUrl;
+        let commitHash = null;
 
-        // Fetch specs.json content
-        const specsJsonResponse = await axios.get(specsJsonUrl, { headers });
-        if (specsJsonResponse.status !== 200) {
-            console.log(`❌ Could not find specs.json in repository ${owner}/${repo}`);
-            return null;
-        }
+        if (options.ghPageUrl) {
+            // Fetch from GitHub Pages (deployed HTML)
+            indexHtmlUrl = options.ghPageUrl.endsWith('/') ? 
+                `${options.ghPageUrl}index.html` : 
+                `${options.ghPageUrl}/index.html`;
+            console.log(`Fetching index.html from GitHub Pages: ${indexHtmlUrl}`);
+            
+            // For GitHub Pages, we'll try to get the commit hash from the main branch
+            try {
+                const mainBranchUrl = `https://api.github.com/repos/${owner}/${repo}/branches/main`;
+                const branchResponse = await axios.get(mainBranchUrl, { headers });
+                if (branchResponse.status === 200) {
+                    commitHash = branchResponse.data.commit.sha;
+                    console.log(`✅ Got commit hash from main branch: ${commitHash}`);
+                }
+            } catch (error) {
+                console.log(`⚠️ Could not get commit hash from main branch: ${error.message}`);
+            }
+        } else {
+            // Fallback to raw repository method
+            console.log(`⚠️ No GitHub Pages URL provided, falling back to repository method`);
+            
+            // Get the specs.json content from the repository to find the output_path
+            const specsJsonUrl = `https://api.github.com/repos/${owner}/${repo}/contents/specs.json`;
+            console.log(`Fetching specs.json from: ${specsJsonUrl}`);
 
-        // Decode specs.json content from base64
-        const specsJsonContent = Buffer.from(specsJsonResponse.data.content, 'base64').toString('utf8');
-        const specsJson = JSON.parse(specsJsonContent);
-        
-        // Get the output_path from specs.json
-        const outputPath = specsJson.specs[0].output_path;
-        if (!outputPath) {
-            console.log(`❌ No output_path found in specs.json for repository ${owner}/${repo}`);
-            return null;
+            // Fetch specs.json content
+            const specsJsonResponse = await axios.get(specsJsonUrl, { headers });
+            if (specsJsonResponse.status !== 200) {
+                console.log(`❌ Could not find specs.json in repository ${owner}/${repo}`);
+                return null;
+            }
+
+            // Decode specs.json content from base64
+            const specsJsonContent = Buffer.from(specsJsonResponse.data.content, 'base64').toString('utf8');
+            const specsJson = JSON.parse(specsJsonContent);
+            
+            // Get the output_path from specs.json
+            const outputPath = specsJson.specs[0].output_path;
+            if (!outputPath) {
+                console.log(`❌ No output_path found in specs.json for repository ${owner}/${repo}`);
+                return null;
+            }
+
+            // Fix: Properly normalize the output path to ensure it doesn't have leading "./" or trailing "/"
+            const normalizedOutputPath = outputPath.replace(/^\.\//, '').replace(/\/$/, '');
+            
+            // Create the path to the index.html file
+            const indexHtmlPath = `${normalizedOutputPath}/index.html`;
+            
+            // Fetch the index.html content with properly constructed URL
+            indexHtmlUrl = `https://raw.githubusercontent.com/${owner}/${repo}/main/${indexHtmlPath}`;
+            console.log(`Fetching index.html from raw repository: ${indexHtmlUrl}`);
+
+            // Get the commit hash for the index.html file
+            commitHash = await getFileCommitHash(token, owner, repo, indexHtmlPath, headers);
+            if (!commitHash) {
+                console.log(`⚠️ Could not get commit hash for index.html, continuing without it`);
+            }
         }
 
-        // Fix: Properly normalize the output path to ensure it doesn't have leading "./" or trailing "/"
-        const normalizedOutputPath = outputPath.replace(/^\.\//, '').replace(/\/$/, '');
-        
-        // Create the path to the index.html file
-        const indexHtmlPath = `${normalizedOutputPath}/index.html`;
-        
-        // Fetch the index.html content with properly constructed URL
-        const indexHtmlUrl = `https://raw.githubusercontent.com/${owner}/${repo}/main/${indexHtmlPath}`;
-        console.log(`Fetching index.html from: ${indexHtmlUrl}`);
-        
+        // Fetch the index.html content
         const indexHtmlResponse = await axios.get(indexHtmlUrl, { headers });
         if (indexHtmlResponse.status !== 200) {
             console.log(`❌ Could not find index.html at ${indexHtmlUrl}`);
             return null;
         }
 
-        // Get the commit hash for the index.html file
-        const commitHash = await getFileCommitHash(token, owner, repo, indexHtmlPath, headers);
-        if (!commitHash) {
-            console.log(`⚠️ Could not get commit hash for index.html, continuing without it`);
-        }
-
         const htmlContent = indexHtmlResponse.data;
         
         // Parse HTML using JSDOM
@@ -257,11 +217,6 @@ async function fetchAllTermsFromIndex(token, owner, repo, options = {}) {
         fs.writeFileSync(outputFilePath, JSON.stringify(result, null, 2));
         console.log(`✅ Saved ${terms.length} terms to ${outputFilePath}`);
         
-        // Save to cache if enabled
-        if (options.cache !== false) {
-            saveToCache(cacheKey, result);
-        }
-        
         return result;
 
     } catch (error) {

package/src/collectExternalReferences/processXTrefsData.js

@@ -4,19 +4,12 @@ const { matchTerm } = require('./matchTerm.js');
 const { addPath, getPath, getAllPaths } = require('../../config/paths');
 const path = require('path');
 
-// Directory to store cached files
+// Directory to store fetched data files
 const CACHE_DIR = getPath('githubcache');
 
-async function processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, outputPathJS, outputPathJSTimeStamped, options) {
+async function processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, outputPathJS, outputPathJSTimeStamped) {
     try {
-        // Clear the cache (remove the cache directory) if the cache option is set to false
-        if (options.cache === false) {
-            if (fs.existsSync(CACHE_DIR)) {
-                fs.rmdirSync(CACHE_DIR, { recursive: true });
-            }
-        }
-        
-        // Ensure the cache directory exists, so that we can store the fetched data
+        // Ensure the directory exists, so that we can store the fetched data
         if (!fs.existsSync(CACHE_DIR)) {
             fs.mkdirSync(CACHE_DIR, { recursive: true });
         }
@@ -51,12 +44,17 @@ async function processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, ou
             const repoGroup = xrefsByRepo[repoKey];
             console.log(`Processing repository: ${repoKey} (${repoGroup.xtrefs.length} terms)`);
             
+            // Get the GitHub Pages URL from the first xtref in this repo group
+            const ghPageUrl = repoGroup.xtrefs[0]?.ghPageUrl;
+            
             // First, fetch all terms from this repository
             const allTermsData = await fetchAllTermsFromIndex(
                 GITHUB_API_TOKEN, 
                 repoGroup.owner, 
                 repoGroup.repo, 
-                options
+                { 
+                    ghPageUrl: ghPageUrl // Pass the GitHub Pages URL
+                }
             );
             
             if (!allTermsData) {

package/src/create-docx.js

@@ -0,0 +1,332 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { JSDOM } = require('jsdom');
+const { Document, Packer, Paragraph, TextRun, HeadingLevel, TableOfContents, Table, TableRow, TableCell, WidthType, AlignmentType } = require('docx');
+
+/**
+ * Creates DOCX metadata from config
+ */
+function createDocxMetadata(config) {
+    return {
+        title: config.specs[0].title || 'Untitled Document',
+        subject: config.specs[0].description || '',
+        creator: config.specs[0].author || '',
+        keywords: config.specs[0].keywords || [],
+        description: config.specs[0].description || '',
+        lastModifiedBy: 'Spec-Up DOCX Generator',
+        revision: 1,
+        createdAt: new Date(),
+        modifiedAt: new Date()
+    };
+}
+
+/**
+ * Converts HTML heading to DOCX heading level
+ */
+function getHeadingLevel(tagName) {
+    const levels = {
+        'h1': HeadingLevel.HEADING_1,
+        'h2': HeadingLevel.HEADING_2,
+        'h3': HeadingLevel.HEADING_3,
+        'h4': HeadingLevel.HEADING_4,
+        'h5': HeadingLevel.HEADING_5,
+        'h6': HeadingLevel.HEADING_6
+    };
+    return levels[tagName.toLowerCase()] || HeadingLevel.HEADING_1;
+}
+
+/**
+ * Processes HTML node and converts to DOCX paragraphs
+ */
+function processNode(node, elements = []) {
+    if (node.nodeType === 3) { // Text node
+        const text = node.textContent.trim();
+        if (text) {
+            elements.push(new Paragraph({
+                children: [new TextRun(text)]
+            }));
+        }
+        return elements;
+    }
+
+    if (node.nodeType !== 1) return elements; // Skip non-element nodes
+
+    const tagName = node.tagName.toLowerCase();
+
+    switch (tagName) {
+        case 'h1':
+        case 'h2':
+        case 'h3':
+        case 'h4':
+        case 'h5':
+        case 'h6':
+            elements.push(new Paragraph({
+                text: node.textContent.trim(),
+                heading: getHeadingLevel(tagName)
+            }));
+            break;
+
+        case 'p':
+            const textRuns = [];
+            processInlineElements(node, textRuns);
+            if (textRuns.length > 0) {
+                elements.push(new Paragraph({
+                    children: textRuns
+                }));
+            }
+            break;
+
+        case 'ul':
+        case 'ol':
+            const listItems = Array.from(node.children);
+            listItems.forEach(li => {
+                if (li.tagName.toLowerCase() === 'li') {
+                    elements.push(new Paragraph({
+                        text: li.textContent.trim(),
+                        bullet: tagName === 'ul' ? { level: 0 } : { level: 0 }
+                    }));
+                }
+            });
+            break;
+
+        case 'table':
+            const tableRows = Array.from(node.querySelectorAll('tr'));
+            if (tableRows.length > 0) {
+                const docxRows = tableRows.map(row => {
+                    const cells = Array.from(row.querySelectorAll('td, th'));
+                    return new TableRow({
+                        children: cells.map(cell => new TableCell({
+                            children: [new Paragraph({
+                                text: cell.textContent.trim()
+                            })],
+                            width: {
+                                size: 100 / cells.length,
+                                type: WidthType.PERCENTAGE
+                            }
+                        }))
+                    });
+                });
+
+                elements.push(new Table({
+                    rows: docxRows,
+                    width: {
+                        size: 100,
+                        type: WidthType.PERCENTAGE
+                    }
+                }));
+            }
+            break;
+
+        case 'blockquote':
+            elements.push(new Paragraph({
+                text: node.textContent.trim(),
+                indent: {
+                    left: 720 // 0.5 inch in twips
+                }
+            }));
+            break;
+
+        case 'dl':
+            // Process definition lists
+            const dlItems = Array.from(node.children);
+            dlItems.forEach(item => {
+                if (item.tagName.toLowerCase() === 'dt') {
+                    elements.push(new Paragraph({
+                        children: [new TextRun({
+                            text: item.textContent.trim(),
+                            bold: true
+                        })]
+                    }));
+                } else if (item.tagName.toLowerCase() === 'dd') {
+                    elements.push(new Paragraph({
+                        text: item.textContent.trim(),
+                        indent: {
+                            left: 360 // 0.25 inch in twips
+                        }
+                    }));
+                }
+            });
+            break;
+
+        default:
+            // For other elements, process their children
+            Array.from(node.childNodes).forEach(child => {
+                processNode(child, elements);
+            });
+            break;
+    }
+
+    return elements;
+}
+
+/**
+ * Processes inline elements within paragraphs
+ */
+function processInlineElements(node, textRuns) {
+    Array.from(node.childNodes).forEach(child => {
+        if (child.nodeType === 3) { // Text node
+            const text = child.textContent;
+            if (text.trim()) {
+                textRuns.push(new TextRun(text));
+            }
+        } else if (child.nodeType === 1) { // Element node
+            const tagName = child.tagName.toLowerCase();
+            const text = child.textContent.trim();
+            
+            if (text) {
+                switch (tagName) {
+                    case 'strong':
+                    case 'b':
+                        textRuns.push(new TextRun({
+                            text: text,
+                            bold: true
+                        }));
+                        break;
+                    case 'em':
+                    case 'i':
+                        textRuns.push(new TextRun({
+                            text: text,
+                            italics: true
+                        }));
+                        break;
+                    case 'code':
+                        textRuns.push(new TextRun({
+                            text: text,
+                            font: 'Courier New'
+                        }));
+                        break;
+                    case 'a':
+                        textRuns.push(new TextRun({
+                            text: text,
+                            color: '0000FF',
+                            underline: {}
+                        }));
+                        break;
+                    default:
+                        textRuns.push(new TextRun(text));
+                        break;
+                }
+            }
+        }
+    });
+}
+
+/**
+ * Creates a title page
+ */
+function createTitlePage(config) {
+    const elements = [];
+    const spec = config.specs[0];
+
+    if (spec.title) {
+        elements.push(new Paragraph({
+            text: spec.title,
+            heading: HeadingLevel.TITLE,
+            alignment: AlignmentType.CENTER
+        }));
+    }
+
+    if (spec.description) {
+        elements.push(new Paragraph({
+            text: spec.description,
+            alignment: AlignmentType.CENTER
+        }));
+    }
+
+    if (spec.author) {
+        elements.push(new Paragraph({
+            text: `Author: ${spec.author}`,
+            alignment: AlignmentType.CENTER
+        }));
+    }
+
+    // Add some spacing
+    elements.push(new Paragraph({ text: '' }));
+    elements.push(new Paragraph({ text: '' }));
+
+    return elements;
+}
+
+(async () => {
+    try {
+        console.log('Starting DOCX generation...');
+
+        // Read and parse the specs.json file
+        const config = fs.readJsonSync('specs.json');
+        const metadata = createDocxMetadata(config);
+
+        // Extract configuration details
+        const outputPath = config.specs[0].output_path;
+        const filePath = path.resolve(process.cwd(), outputPath, 'index.html');
+
+        // Check if HTML file exists
+        if (!fs.existsSync(filePath)) {
+            console.error(`❌ HTML file not found at ${filePath}`);
+            console.log('Please run "npm run render" first to generate the HTML file.');
+            return;
+        }
+
+        // Read and parse the HTML file
+        const htmlContent = fs.readFileSync(filePath, 'utf8');
+        const dom = new JSDOM(htmlContent);
+        const document = dom.window.document;
+
+        // Remove unnecessary elements
+        document.querySelectorAll('script, style, .d-print-none, [style*="display: none"]').forEach(el => el.remove());
+
+        // Start building the DOCX document
+        const docElements = [];
+
+        // Add title page
+        docElements.push(...createTitlePage(config));
+
+        // Add table of contents placeholder
+        docElements.push(new TableOfContents('Table of Contents', {
+            hyperlink: true,
+            headingStyleRange: '1-6'
+        }));
+
+        // Add page break after TOC
+        docElements.push(new Paragraph({
+            text: '',
+            pageBreakBefore: true
+        }));
+
+        // Process the main content
+        const mainContent = document.querySelector('main') || document.body;
+        processNode(mainContent, docElements);
+
+        // Create the DOCX document
+        const doc = new Document({
+            properties: {
+                title: metadata.title,
+                subject: metadata.subject,
+                creator: metadata.creator,
+                keywords: metadata.keywords.join(', '),
+                description: metadata.description,
+                lastModifiedBy: metadata.lastModifiedBy,
+                revision: metadata.revision,
+                createdAt: metadata.createdAt,
+                modifiedAt: metadata.modifiedAt
+            },
+            sections: [{
+                children: docElements
+            }]
+        });
+
+        // Generate the DOCX file
+        const buffer = await Packer.toBuffer(doc);
+        const docxPath = path.resolve(process.cwd(), 'docs/index.docx');
+
+        // Ensure docs directory exists
+        fs.ensureDirSync(path.dirname(docxPath));
+
+        // Write the DOCX file
+        fs.writeFileSync(docxPath, buffer);
+
+        console.log('✅ DOCX generated successfully! Find the DOCX file in the docs directory.');
+    } catch (error) {
+        console.error('❌ Error generating DOCX:', error);
+        process.exit(1);
+    }
+})();