spec-up-t 1.0.6 → 1.0.8

package/src/get-xrefs-data.js

@@ -5,285 +5,204 @@
  * @since 2024-06-09
  */
 
+
 const fs = require('fs-extra');
 const config = fs.readJsonSync('specs.json');
 
 // Collect all directories that contain files with a term and definition
+// This maps over the specs in the config file and constructs paths to directories
+// where the term definition files are located.
 const specTermsDirectories = config.specs.map(spec => spec.spec_directory + '/' + spec.spec_terms_directory);
 
-// Create directory named “output” in the project root if it does not yet exist
+// Ensure that the 'output' directory exists, creating it if necessary.
 if (!fs.existsSync('output')) {
     fs.mkdirSync('output');
 }
 
-// Create directory named “output/xrefs-history” in the project root if it does not yet exist
+// Ensure that the 'output/xrefs-history' directory exists, creating it if necessary.
 if (!fs.existsSync('output/xrefs-history')) {
     fs.mkdirSync('output/xrefs-history');
 }
 
-// Create a path for the output file in the project root
+// Define paths for various output files, including JSON and JS files.
 const outputPathJSON = 'output/xrefs-data.json';
 const outputPathJS = 'output/xrefs-data.js';
 const outputPathJSTimeStamped = 'output/xrefs-history/xrefs-data-' + Date.now() + '.js';
 
-function getXrefsData() {
-    let allXrefs = {};
-    allXrefs.xrefs = new Set();
-
-    // Function to fetch the latest commit hash of the file
-    async function fetchLatestCommitHash(match) {
-        /* Example
-            console.log('match: ', match); ->
-            
-            match:  {
-                externalSpec: 'test-1',
-                term: 'Aal',
-                repoUrl: 'https://github.com/blockchainbird/spec-up-xref-test-1',
-                terms_dir: 'spec/term-definitions',
-                owner: 'blockchainbird',
-                repo: 'spec-up-xref-test-1',
-                site: 'https://blockchainbird.github.io/spec-up-xref-test-1/'
-            }
-        */
 
-        
-        
-        try {
+function getXrefsData(GITHUB_API_TOKEN) {
+    // Set headers for GitHub API requests. Include an authorization token if provided.
+    const fetchHeaders = {
+        'Accept': 'application/vnd.github.v3+json'
+    };
 
-            if (match.repoUrl === undefined) {
-                console.log('\n   SPEC-UP-T: match.repoUrl is undefined' + "\n");
-                return;
-            }
+    if (GITHUB_API_TOKEN) {
+        fetchHeaders['Authorization'] = `token ${GITHUB_API_TOKEN}`;
+    } else {
+        console.log('\n   SPEC-UP-T: There is no GitHub token set up. Therefore, you are more likely to be at your limit of GitHub API requests. If you run into the limit, create a token and search the documentation on this topic.\n');
+    }
 
-            // prerequisite: filename should be the term in the match object with spaces replaced by dashes and all lowercase
-            const url = `https://api.github.com/repos/${match.owner}/${match.repo}/commits?path=${match.terms_dir}/${match.term.replace(/ /g, '-').toLowerCase()}.md`;
+    // Function to check the rate limit of the GitHub API
+    function checkRateLimit(response) {
+        if (response.status === 403 && response.headers.get('X-RateLimit-Remaining') === '0') {
+            const resetTime = new Date(response.headers.get('X-RateLimit-Reset') * 1000);
+            console.error(`\n   SPEC-UP-T: Github API rate limit exceeded. Try again after ${resetTime}. See https://blockchainbird.github.io/spec-up-t-website/docs/github-token/ for more info.` + "\n");
+            return true;
+        } else {
+            console.log(`\n   SPEC-UP-T: Github API rate limit: ${response.headers.get('X-RateLimit-Remaining')} requests remaining. See https://blockchainbird.github.io/spec-up-t-website/docs/github-token/ for more info.` + "\n");
+        }
+        return false;
+    }
 
-            // Fetch the list of commits for the specified file
-            const response = await fetch(url, {
-                headers: {
-                    'Accept': 'application/vnd.github.v3+json'
-                }
-            });
+    // Function to fetch term information from GitHub, including commit hash and content.
+    async function fetchTermInfoFromGithub(xref) {
+        try {
+            // prerequisite: filename should be the term in the match object with spaces replaced by dashes and all lowercase
+            //TODO: Loop through all markdown files to find the term and get the filename, instead of assuming that the filename is the term with spaces replaced by dashes and all lowercase
+            const url = `https://api.github.com/repos/${xref.owner}/${xref.repo}/commits?path=${xref.terms_dir}/${xref.term.replace(/ /g, '-').toLowerCase()}.md&per_page=1`;
+            const response = await fetch(url, { headers: fetchHeaders });
 
             // Check for rate limit before proceeding
-            if (response.status === 403 && response.headers.get('X-RateLimit-Remaining') === '0') {
-                const resetTime = new Date(response.headers.get('X-RateLimit-Reset') * 1000);
-                console.error(`\n   SPEC-UP-T: Github API rate limit exceeded. Try again after ${resetTime}` + "\n");
+            if (checkRateLimit(response)) {
                 return;
             }
 
-            // Check if the request was successful
-            if (!response.ok) {
-                throw new Error(`HTTP error! status: ${response.status}`);
-            }
-
-            console.log(`\n   SPEC-UP-T: Github API request for:\n   Term ${match.term},\n   Name: ${match.externalSpec}\n   Owner ${match.owner}\n   Repo ${match.repo}\n   was successful` + "\n");
-
-            // Extract JSON data from the response, see https://blockchainbird.github.io/spec-up-t-website/docs/various-roles/developers-guide/#example-of-api-response for example response
-            const data = await response.json();
-
-            // Check if there are any commits
-            if (data.length === 0) {
-                console.log(`\n   SPEC-UP-T: No commit hash found for the term “${match.term}”` + "\n");
-
-                return;
+            if (response.ok) {
+                const data = await response.json();
+                if (data.length > 0) {
+                    const commitHash = data[0].sha;
+                    const content = await fetchFileContentFromCommit(xref.owner, xref.repo, commitHash, `${xref.terms_dir}/${xref.term.replace(/ /g, '-').toLowerCase()}.md`);
+                    return { commitHash, content };
+                }
+            } else {
+                console.error(`\n   SPEC-UP-T: Failed to fetch commit hash for ${xref.term}: ${response.statusText}\n`);
+                return { commitHash: null, content: null };
             }
+        } catch (error) {
+            console.error(`\n   SPEC-UP-T: Error fetching data for term ${xref.term}: ${error.message}\n`);
+        }
+        return null;
+    }
 
-            // Get only the last commit
-            const commits = data.slice(0, 1);
-            // Assign the fetched commit hash to the variable commitHash
-            let commitHash = commits.map(commit => commit.sha);
-            
-            console.log(`\n   SPEC-UP-T: Commit hash found for the term “${match.term}”: `, commitHash + "\n");
-
-
-            //TODO: Check if a term is in the JSON file and not in the markdown file. If so, remove the term from the JSON file.
-
-            // Check if the file exists
-            if (fs.existsSync(outputPathJSON)) {
-                // Read the JSON file
-                let currentXrefs = fs.readJsonSync(outputPathJSON);
-                // Check if the term is in the JSON file
-                currentXrefs.xrefs.forEach(xref => {
-                    // Check if the term is in the JSON file
-                    if (xref.term === match.term) {
-                        // If the term is in the JSON file, get the commit hash from the file and assign it to the variable commitHash. This is done to prevent the commit hash from being overwritten by the fetched commit hash. We want to keep the commit hash that was fetched at the time that the author looked it up.
-                        console.log(`\n   SPEC-UP-T: This external reference:\n Term: ${match.term}\n Name: ${match.externalSpec}\n Owner: ${match.owner}\n Repo: ${match.repo}\nis already referenced.
-                        ` + "\n")
-
-                        // Give the commitHash from the JSON file to the commitHash variable
-                        commitHash = xref.commitHash;
+    // Function to extend xref objects with additional information, such as repository URL and directory information.
+    function extendXrefs(config, xrefs) {
+        xrefs.forEach(xref => {
+            config.specs.forEach(spec => {
+                // Loop through "external_specs_repos" to find the repository URL for each xref
+                xref.repoUrl = null;
+                xref.terms_dir = null;
+                xref.owner = null;
+                xref.repo = null;
+
+                spec.external_specs_repos.forEach(repo => {
+                    if (repo.external_spec === xref.externalSpec) {
+                        xref.repoUrl = repo.url;
+                        xref.terms_dir = repo.terms_dir;
+                        const urlParts = new URL(xref.repoUrl).pathname.split('/');
+                        xref.owner = urlParts[1];
+                        xref.repo = urlParts[2];
                     }
                 });
-            } else {
-                console.log(`\n   SPEC-UP-T: File not found at this point: ${outputPathJSON}. Don't worry, it will be created later.` + "\n");
-            }
 
-            return commitHash;
+                // Loop through "external_specs" to find the site URL for each xref
+                
+                xref.site = null;
+                if (spec.external_specs) {
+                    spec.external_specs.forEach(externalSpec => {
+                        const key = Object.keys(externalSpec)[0];
+                        if (key === xref.externalSpec) {
+                            xref.site = externalSpec[key];
+                        }
+                    });
+                }
+            });
+        });
+    }
+
+    // Function to fetch the content of a file from a specific commit in a GitHub repository.
+    async function fetchFileContentFromCommit(owner, repo, commitHash, filePath) {
+        try {
+            const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${commitHash}?recursive=1`;
+            const treeResponse = await fetch(treeUrl, { headers: fetchHeaders });
+
+            if (treeResponse.ok) {
+                const treeData = await treeResponse.json();
+                const file = treeData.tree.find(item => item.path === filePath);
+                if (file) {
+                    const fileContentResponse = await fetch(file.url);
+                    const fileContentData = await fileContentResponse.json();
+                    return Buffer.from(fileContentData.content, 'base64').toString('utf-8');
+                }
+            }
         } catch (error) {
-            console.error(`\n   SPEC-UP-T: Failed to fetch commit hash for the term “${match.term}”:`, error + "\n");
+            console.error(`\n   SPEC-UP-T: Error fetching file content: ${error.message}\n`);
         }
+        return null;
     }
 
-    async function fetchLatestCommitHashes() {
-        for (const xref of allXrefs.xrefs) {
-            xref.commitHash = await fetchLatestCommitHash(xref);
-        }
+    // Initialize an object to store all xrefs. If the output JSON file exists, load its data.
+    let allXrefs = { xrefs: [] };
+
+    if (fs.existsSync(outputPathJSON)) {
+        const existingXrefs = fs.readJsonSync(outputPathJSON);
+        allXrefs = existingXrefs && existingXrefs.xrefs ? existingXrefs : { xrefs: [] };
     }
 
-    // Go through all directories that contain files with a term and definition
+    // Loop through each directory and file, extracting xrefs from markdown files.
     specTermsDirectories.forEach(specDirectory => {
-        console.log(`\n   SPEC-UP-T: Current spec_directory: `, specDirectory + "\n");
-        // read directory
         fs.readdirSync(specDirectory).forEach(file => {
-            // read file
             if (file.endsWith('.md')) {
-                console.log(`\n   SPEC-UP-T: Markdown file referenced in spec_directory: `, file + "\n");
                 const markdown = fs.readFileSync(`${specDirectory}/${file}`, 'utf8');
-                // create regex that finds “[[xref:.*]]”
                 const regex = /\[\[xref:.*?\]\]/g;
                 if (regex.test(markdown)) {
                     const xrefs = markdown.match(regex);
                     xrefs.forEach(xref => {
-                        console.log(`\n   SPEC-UP-T: Xref found in ${file}: `, xref + "\n");
-                        // example of xref: [xref: test-1, Aal]
-                        allXrefs.xrefs.add(xref);
+                        const newXrefObj = processXref(xref);
+                        if (!allXrefs.xrefs.some(existingXref =>
+                            existingXref.term === newXrefObj.term && existingXref.externalSpec === newXrefObj.externalSpec)) {
+                            allXrefs.xrefs.push(newXrefObj);
+                        }
                     });
                 }
             }
         });
-    })
-    // Convert the Set back to an Array if needed
-    allXrefs.xrefs = Array.from(allXrefs.xrefs);
-
-    // Example output:
-    // allXrefs.xrefs: [
-    //     '[[xref: test-1, Aal]]',
-    //     '[[xref: test-2, Abac]]'
-    // ]
-
-    // The following steps create an array of objects with the keys “externalSpec” and “term” for each xref by splitting the xref string on the comma and removing the “[[xref:” and “]]” parts
-
-    // Step 1: remove “[[xref:” from the beginning of every value in allMatches
-    allXrefs.xrefs = allXrefs.xrefs.map(xref => {
-        return xref.replace(/\[\[xref:/, '');
-    });
-
-    // Step 2: remove “]]” from the end of every value in allMatches
-    allXrefs.xrefs = allXrefs.xrefs.map(xref => {
-        return xref.replace(/\]\]/, '');
     });
 
-    // Step 3: trim every entry of allMatches
-    allXrefs.xrefs = allXrefs.xrefs.map(xref => {
-        return xref.trim();
-    });
-
-    // Step 4: split every entry of allMatches on the first comma, replace the entry with an object that has two keys: one that contains everything before the comma and one that contains everything after the comma
-    allXrefs.xrefs = allXrefs.xrefs.map(xref => {
-        let [externalSpec, term] = xref.split(/,/, 2);
+    // Function to process and clean up xref strings, returning an object with `externalSpec` and `term` properties.
+    function processXref(xref) {
+        let [externalSpec, term] = xref.replace(/\[\[xref:/, '').replace(/\]\]/, '').trim().split(/,/, 2);
         return {
             externalSpec: externalSpec.trim(),
             term: term.trim()
         };
-    })
-
-    // Example output:
-    // allXrefs.xrefs: [
-    //     { externalSpec: 'test-1', term: 'Aal' },
-    //     { externalSpec: 'test-2', term: 'Abac' }
-    // ]
-
-    // Step 5: add the url and the dir where the terms are, to the xref object
-    allXrefs.xrefs.forEach(xref => {
-        config.specs.forEach(spec => {
-            spec.external_specs_repos.forEach(repo => {
-                // if the externalSpec is in the config, add the url and the dir where the terms are, to the xref object
-                // Example external_specs_repos:
-                // "external_specs_repos": [
-                //     {
-                //         "external_spec": "test-1",
-                //         "url": "https://github.com/blockchainbird/spec-up-xref-test-1",
-                //         "terms_dir": "spec"
-                //     },
-                // …
-                // ]
-                if (repo.external_spec === xref.externalSpec) {
-                    xref.repoUrl = repo.url;
-                    xref.terms_dir = repo.terms_dir;
-                }
-            });
-        });
-    });
-
-    // Step 6: add the owner and repo to the xref object
-    allXrefs.xrefs.forEach(xref => {
-        if (xref.repoUrl === undefined) {
-            console.log('\n   SPEC-UP-T: match.repoUrl is undefined' + "\n");
-            return;
-        }
+    }
 
-        const urlParts = new URL(xref.repoUrl).pathname.split('/');
-        xref.owner = urlParts[1];
-        xref.repo = urlParts[2];
-    });
+    // Extend each xref with additional data and fetch commit information from GitHub.
+    extendXrefs(config, allXrefs.xrefs);
 
-    // Step 7: add the site to the xref object
-    allXrefs.xrefs.forEach(xref => {
-        // loop through array of specs in config
-        config.specs.forEach(spec => {
-            if (spec.external_specs) {
-                // Example external_specs:
-                // "external_specs": [
-                //      {
-                //          "test-1": "https://blockchainbird.github.io/spec-up-xref-test-1/"
-                //      }
-                // …
-                // ]
-                spec.external_specs.forEach(externalSpec => {
-                    const key = Object.keys(externalSpec)[0];
-                    if (key === xref.externalSpec) {
-                        xref.site = externalSpec[key];
-                    }
-                });
+    async function fetchAllTermsInfoFromGithub() {
+        for (let xref of allXrefs.xrefs) {
+            if (!xref.commitHash || !xref.content) {
+                const fetchedData = await fetchTermInfoFromGithub(xref);
+                if (fetchedData) {
+                    xref.commitHash = fetchedData.commitHash;
+                    xref.content = fetchedData.content;
+                }
             }
-        });
-    });
-
-    // Loop through all xrefs and fetch the latest commit hash for each term and add it to the xref object
-    /* Example of xref after adding commitHash:
-        xref:  {
-            "externalSpec": "test-1",
-            "term": "Aal",
-            "repoUrl": "https://github.com/blockchainbird/spec-up-xref-test-1",
-            "terms_dir": "spec/term-definitions",
-            "owner": "blockchainbird",
-            "repo": "spec-up-xref-test-1",
-            "site": "https://blockchainbird.github.io/spec-up-xref-test-1/",
-            "commitHash": [
-                "f66951f1d378490289caab9c51141b44a0438365"
-            ]
         }
-    */
+    }
 
-    // Call the function and wait for it to complete before writing to the file
-    fetchLatestCommitHashes().then(() => {
-        // Convert allXrefsStr to a JSON string with indentation
+    // Fetch all term information, then write the results to JSON and JS files.
+    fetchAllTermsInfoFromGithub().then(() => {
         const allXrefsStr = JSON.stringify(allXrefs, null, 2);
-
-        // // Write the JSON code to a .json file
         fs.writeFileSync(outputPathJSON, allXrefsStr, 'utf8');
-
-        // Create the JS code for the assignment
         const stringReadyForFileWrite = `const allXrefs = ${allXrefsStr};`;
-
-        // Write the JS code to a .js file
         fs.writeFileSync(outputPathJS, stringReadyForFileWrite, 'utf8');
         fs.writeFileSync(outputPathJSTimeStamped, stringReadyForFileWrite, 'utf8');
     });
 }
 
-// Write function that removes an entry from xrefs-data.json and xrefs-data.js based on the term and externalSpec
+// Function to remove a specific xref from the JSON file, based on term and externalSpec.
 function removeXref(term, externalSpec) {
     let messages = [];
 
@@ -325,8 +244,8 @@ function removeXref(term, externalSpec) {
     return messages;
 }
 
-
+// Export the getXrefsData and removeXref functions for use in other modules.
 module.exports = {
     getXrefsData,
     removeXref
-}
+}