spec-up-t 1.0.88 → 1.0.90

package/src/get-xtrefs-data/searchGitHubCode.js

@@ -0,0 +1,97 @@
+async function searchGitHubCode(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
+    const { Octokit } = await import("octokit");
+    const { throttling } = await import("@octokit/plugin-throttling");
+
+    // Create a throttled Octokit instance
+    const ThrottledOctokit = Octokit.plugin(throttling);
+    const octokit = new ThrottledOctokit({
+        auth: GITHUB_API_TOKEN,
+        throttle: {
+            onRateLimit: (retryAfter, options) => {
+                console.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+            onAbuseLimit: (retryAfter, options) => {
+                console.warn(`Abuse detected for request ${options.method} ${options.url}`);
+            },
+            onSecondaryRateLimit: (retryAfter, options) => {
+                console.warn(`Secondary rate limit hit for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+        },
+    });
+
+    try {
+        // Perform the search using Octokit with exact match
+        const searchResponse = await octokit.rest.search.code({
+            q: `"${searchString}" repo:${owner}/${repo} path:${subdirectory}`, // Exact match in subdirectory
+            headers: {
+                Accept: "application/vnd.github.v3.text-match+json", // Include text-match media type
+            },
+        });
+
+        // Log the search results
+        console.log("Total results:", searchResponse.data.total_count);
+
+        // Fetch the content of each file
+        for (const item of searchResponse.data.items) {
+            // Check if text_matches exists and is not empty
+            if (!item.text_matches || item.text_matches.length === 0) {
+                console.log(`Skipping ${item.path}: No text matches found.`);
+                continue;
+            }
+
+            // Check if the match is in the first line using text_matches
+            const isFirstLineMatch = item.text_matches.some(match => {
+                if (!match.fragment) {
+                    console.log(`Skipping ${item.path}: No fragment found in text match.`);
+                    return false;
+                }
+
+                const firstLine = match.fragment.split("\n")[0];
+                return firstLine.includes(searchString);
+            });
+
+            if (!isFirstLineMatch) {
+                console.log(`Skipping ${item.path}: Match not in the first line.`);
+                continue; // Skip this file
+            }
+
+            // Fetch file content
+            let content = "";
+            try {
+                const fileContentResponse = await octokit.rest.repos.getContent({
+                    owner: item.repository.owner.login, // Repository owner
+                    repo: item.repository.name, // Repository name
+                    path: item.path, // File path
+                });
+
+                // Decode the file content (it's base64-encoded)
+                if (fileContentResponse.data.content) {
+                    content = Buffer.from(fileContentResponse.data.content, "base64").toString("utf-8");
+                } else {
+                    // If the file is larger than 1 MB, GitHub's API will return a download URL instead of the content.
+                    console.log("File is too large. Download URL:", fileContentResponse.data.download_url);
+                }
+            } catch (error) {
+                console.error(`Error fetching content for ${item.path}:`, error);
+                content = ""; // Set content to an empty string if there's an error
+            }
+
+            // Attach the content to the item
+            item.content = content;
+        }
+
+        return searchResponse;
+    } catch (error) {
+        console.error("Error searching GitHub or fetching file content:", error);
+    }
+}
+
+exports.searchGitHubCode = searchGitHubCode;

package/src/get-xtrefs-data/setupFetchHeaders.js

@@ -0,0 +1,14 @@
+function setupFetchHeaders(GITHUB_API_TOKEN) {
+    const fetchHeaders = {
+        'Accept': 'application/vnd.github.v3+json'
+    };
+
+    if (GITHUB_API_TOKEN) {
+        fetchHeaders['Authorization'] = `token ${GITHUB_API_TOKEN}`;
+    } else {
+        console.log('\n   SPEC-UP-T: There is no GitHub token set up. Therefore, you are more likely to be at your limit of GitHub API requests. If you run into the limit, create a token and search the documentation on this topic.\n');
+    }
+
+    return fetchHeaders;
+}
+exports.setupFetchHeaders = setupFetchHeaders;

package/src/get-xtrefs-data.js

@@ -10,9 +10,33 @@
  * @since 2024-06-09
  */
 
-
 const fs = require('fs-extra');
+const path = require('path');
+const readlineSync = require('readline-sync');
+const { searchGitHubCode } = require('./get-xtrefs-data/searchGitHubCode.js');
+const { matchTerm } = require('./get-xtrefs-data/matchTerm');
 const config = fs.readJsonSync('specs.json');
+const { doesUrlExist } = require('./utils/doesUrlExist.js');
+const { addPath, getPath, getAllPaths } = require('./config/paths');
+
+const externalSpecsRepos = config.specs[0].external_specs;
+
+// Check if the URLs for the external specs repositories are valid, and prompt the user to abort if they are not.
+externalSpecsRepos.forEach(repo => {
+    // Construct the URL for the terms directory of the repository
+
+    doesUrlExist(repo.url, repo.terms_dir).then(exists => {
+        if (!exists) {
+            const userInput = readlineSync.question(`\n   SPEC-UP-T: This external reference is not a valid URL:\n   Repository: ${repo.url},\n   Terms directory: ${repo.terms_dir}\n   Do you want to stop? (yes/no): `);
+            if (userInput.toLowerCase() === 'yes' || userInput.toLowerCase() === 'y') {
+                console.log('Stopping...');
+                process.exit(1);
+            }
+        }
+    }).catch(error => {
+        console.error('\n   SPEC-UP-T:Error checking URL existence:', error);
+    });
+});
 
 // Collect all directories that contain files with a term and definition
 // This maps over the specs in the config file and constructs paths to directories
@@ -34,110 +58,11 @@ const outputPathJSON = 'output/xtrefs-data.json';
 const outputPathJS = 'output/xtrefs-data.js';
 const outputPathJSTimeStamped = 'output/xtrefs-history/xtrefs-data-' + Date.now() + '.js';
 
-function setupFetchHeaders(GITHUB_API_TOKEN) {
-    const fetchHeaders = {
-        'Accept': 'application/vnd.github.v3+json'
-    };
-
-    if (GITHUB_API_TOKEN) {
-        fetchHeaders['Authorization'] = `token ${GITHUB_API_TOKEN}`;
-    } else {
-        console.log('\n   SPEC-UP-T: There is no GitHub token set up. Therefore, you are more likely to be at your limit of GitHub API requests. If you run into the limit, create a token and search the documentation on this topic.\n');
-    }
-
-    return fetchHeaders;
-}
-
-// Function to check the rate limit of the GitHub API
-function checkRateLimit(response) {
-    if (response.status === 403 && response.headers.get('X-RateLimit-Remaining') === '0') {
-        const resetTime = new Date(response.headers.get('X-RateLimit-Reset') * 1000);
-        console.error(`\n   SPEC-UP-T: Github API rate limit exceeded. Try again after ${resetTime}. See https://trustoverip.github.io/spec-up-t-website/docs/github-token/ for more info.` + "\n");
-        return true;
-    } else {
-        console.log(`\n   SPEC-UP-T: Github API rate limit: ${response.headers.get('X-RateLimit-Remaining')} requests remaining. See https://trustoverip.github.io/spec-up-t-website/docs/github-token/ for more info.` + "\n");
-    }
-    return false;
-}
-
-// Function to fetch term information from GitHub, including commit hash and content.
-async function fetchTermInfoFromGithub(GITHUB_API_TOKEN, xtref) {
-    try {
-        // prerequisite: filename should be the term in the match object with spaces replaced by dashes and all lowercase
-        //TODO: Loop through all markdown files to find the term and get the filename, instead of assuming that the filename is the term with spaces replaced by dashes and all lowercase
-        const url = `https://api.github.com/repos/${xtref.owner}/${xtref.repo}/commits?path=${xtref.terms_dir}/${xtref.term.replace(/ /g, '-').toLowerCase()}.md&per_page=1`;
-        const response = await fetch(url, { headers: setupFetchHeaders(GITHUB_API_TOKEN) });
-
-        // Check for rate limit before proceeding
-        if (checkRateLimit(response)) {
-            return;
-        }
-
-        if (response.ok) {
-            const data = await response.json();
-            if (data.length > 0) {
-                const commitHash = data[0].sha;
-                const content = await fetchFileContentFromCommit(GITHUB_API_TOKEN, xtref.owner, xtref.repo, commitHash, `${xtref.terms_dir}/${xtref.term.replace(/ /g, '-').toLowerCase()}.md`);
-                return { commitHash, content };
-            }
-        } else {
-            console.error(`\n   SPEC-UP-T: Failed to fetch commit hash for ${xtref.term}: ${response.statusText}\n`);
-            return { commitHash: null, content: null };
-        }
-    } catch (error) {
-        console.error(`\n   SPEC-UP-T: Error fetching data for term ${xtref.term}: ${error.message}\n`);
-    }
-    return null;
-}
-
-// Function to fetch the content of a file from a specific commit in a GitHub repository.
-async function fetchFileContentFromCommit(GITHUB_API_TOKEN, owner, repo, commitHash, filePath) {
-    const MAX_RETRIES = 3;
-    let retries = 0;
-
-    while (retries < MAX_RETRIES) {
-        try {
-            const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${commitHash}?recursive=1`;
-            const treeResponse = await fetch(treeUrl, { headers: setupFetchHeaders(GITHUB_API_TOKEN) });
-
-            if (treeResponse.ok) {
-                const treeData = await treeResponse.json();
-                const file = treeData.tree.find(item => item.path === filePath);
-                if (file) {
-                    const fileContentResponse = await fetch(file.url);
-                    const fileContentData = await fileContentResponse.json();
-                    if (fileContentData.content) {
-                        return Buffer.from(fileContentData.content, 'base64').toString('utf-8');
-                    } else {
-                        console.error('Error: fileContentData.content is undefined');
-                    }
-                } else {
-                    console.error(`Error: File ${filePath} not found in commit ${commitHash}`);
-                }
-            } else if (treeResponse.status === 403 && treeResponse.headers.get('X-RateLimit-Remaining') === '0') {
-                const resetTime = treeResponse.headers.get('X-RateLimit-Reset');
-                const waitTime = resetTime ? (resetTime - Math.floor(Date.now() / 1000)) * 1000 : 60000;
-                console.warn(`Rate limit exceeded. Retrying in ${waitTime / 1000} seconds...`);
-                await new Promise(resolve => setTimeout(resolve, waitTime));
-                retries++;
-            } else {
-                console.error(`Error: Failed to fetch tree from ${treeUrl}`);
-                break;
-            }
-        } catch (error) {
-            console.error(`Error fetching file content: ${error.message}`);
-            break;
-        }
-    }
-
-    return null;
-}
-
 function updateXTrefs(GITHUB_API_TOKEN, skipExisting) {
     // Function to extend xtref objects with additional information, such as repository URL and directory information.
     function extendXTrefs(config, xtrefs) {
         if (config.specs[0].external_specs_repos) {
-            console.log("\n   SPEC-UP-T: PLEASE NOTE: Your specs.json file is outdated (not your fault, we changed something). Use this one: https://github.com/trustoverip/spec-up-t-starter-pack/blob/main/spec-up-t-starterpack/specs.json or e-mail kor@dwarshuis.com for help. \n");
+            console.log("\n   SPEC-UP-T: PLEASE NOTE: Your specs.json file is outdated (not your fault, we changed something). Use this one: https://github.com/trustoverip/spec-up-t-starter-pack/blob/main/spec-up-t-boilerplate/specs.json or e-mail kor@dwarshuis.com for help. \n");
             return;
         }
 
@@ -174,6 +99,23 @@ function updateXTrefs(GITHUB_API_TOKEN, skipExisting) {
         });
     }
 
+    // Function to check if an xtref is in the markdown content
+    function isXTrefInMarkdown(xtref, markdownContent) {
+        // const regex = new RegExp(`\\[\\[xref:${xref.term}\\]\\]`, 'g');
+        const regex = new RegExp(`\\[\\[(?:x|t)ref:${xtref.term}\\]\\]`, 'g');
+        const result = regex.test(markdownContent);
+        return result;
+    }
+
+    // Function to process and clean up xref / tref strings found in the markdown file, returning an object with `externalSpec` and `term` properties.
+    function processXTref(xtref) {
+        let [externalSpec, term] = xtref.replace(/\[\[(?:xref|tref):/, '').replace(/\]\]/, '').trim().split(/,/, 2);
+        return {
+            externalSpec: externalSpec.trim(),
+            term: term.trim()
+        };
+    }
+
     // Initialize an object to store all xtrefs.
     let allXTrefs = { xtrefs: [] };
 
@@ -183,17 +125,10 @@ function updateXTrefs(GITHUB_API_TOKEN, skipExisting) {
         allXTrefs = existingXTrefs && existingXTrefs.xtrefs ? existingXTrefs : { xtrefs: [] };
     }
 
-    // Function to check if an xtref is in the markdown content
-    function isXTrefInMarkdown(xtref, markdownContent) {
-        // const regex = new RegExp(`\\[\\[xref:${xref.term}\\]\\]`, 'g');
-        const regex = new RegExp(`\\[\\[(?:x|t)ref:${xtref.term}\\]\\]`, 'g');
-        const result = regex.test(markdownContent);
-        return result;
-    }
-
     // Collect all markdown content
     let allMarkdownContent = '';
 
+    // Read all main repo Markdown files from a list of directories and concatenate their content into a single string.
     specTermsDirectories.forEach(specDirectory => {
         fs.readdirSync(specDirectory).forEach(file => {
             if (file.endsWith('.md')) {
@@ -210,10 +145,13 @@ function updateXTrefs(GITHUB_API_TOKEN, skipExisting) {
 
     // Add new entries if they are in the markdown
     const regex = /\[\[(?:xref|tref):.*?\]\]/g;
+
+    // `regex` is the regular expression object, and `allMarkdownContent` is the string being tested. The test method returns a boolean value: true if the pattern is found within the string, and false otherwise.
     if (regex.test(allMarkdownContent)) {
         const xtrefs = allMarkdownContent.match(regex);
         xtrefs.forEach(xtref => {
             const newXTrefObj = processXTref(xtref);
+            // Ensure that newXTrefObj is only added to the xtrefs array if there isn't already an object with the same term and externalSpec properties. This helps maintain the uniqueness of entries in the array based on these two properties.
             if (!allXTrefs.xtrefs.some(existingXTref =>
                 existingXTref.term === newXTrefObj.term && existingXTref.externalSpec === newXTrefObj.externalSpec)) {
                 allXTrefs.xtrefs.push(newXTrefObj);
@@ -221,48 +159,62 @@ function updateXTrefs(GITHUB_API_TOKEN, skipExisting) {
         });
     };
 
-    // Function to process and clean up xref / tref strings found in the markdown file, returning an object with `externalSpec` and `term` properties.
-    function processXTref(xtref) {
-        let [externalSpec, term] = xtref.replace(/\[\[(?:xref|tref):/, '').replace(/\]\]/, '').trim().split(/,/, 2);
-        return {
-            externalSpec: externalSpec.trim(),
-            term: term.trim()
-        };
-    }
+    // Example at this point:
+    // allXTrefs.xtrefs: [
+    //     { externalSpec: 'kmg-1', term: 'authentic-chained-data-container' },
+    // ]
 
     // Extend each xref with additional data and fetch commit information from GitHub.
     extendXTrefs(config, allXTrefs.xtrefs);
 
-
-    /* 
-        Function to fetch all term information from GitHub. The function will not fetch the commit hash again if an entry already contains a commit hash.
-
-        It checks if the xtref object already has a commitHash and content.If both are present, it skips fetching the term information from GitHub. This ensures that existing commit hashes are not overwritten.
-    */
-    async function fetchAllTermsInfoFromGithub(skipExisting) {
-        for (let xtref of allXTrefs.xtrefs) {
-            if (!skipExisting || (!xtref.commitHash || !xtref.content)) {
-                const fetchedData = await fetchTermInfoFromGithub(GITHUB_API_TOKEN, xtref);
-                if (fetchedData) {
-                    xtref.commitHash = fetchedData.commitHash;
-                    xtref.content = fetchedData.content;
+    // Example at this point:
+    // allXTrefs.xtrefs: [
+    //     {
+    //         externalSpec: 'kmg-1',
+    //         term: 'authentic-chained-data-container',
+    //         repoUrl: 'https://github.com/henkvancann/keri-main-glossary',
+    //         terms_dir: 'spec/terms-definitions',
+    //         owner: 'henkvancann',
+    //         repo: 'keri-main-glossary',
+    //         site: null
+    //     }
+    // ]
+
+    (async () => {
+        try {
+            for (let xtref of allXTrefs.xtrefs) {
+                const fetchedData = await searchGitHubCode(GITHUB_API_TOKEN, xtref.term, xtref.owner, xtref.repo, xtref.terms_dir);
+                if (fetchedData.data.items.length === 0) {
+                    xtref.commitHash = "not found";
+                    xtref.content = "This term was not found in the external repository.";
+                } else {
+                    fetchedData.data.items.forEach(item => {
+                        // If the term is found according to the matchTerm function (in the first line, line should start with “[[def:), etc) add the commit hash and content to the xtref object
+                        if (matchTerm(item.content, xtref.term)) {
+                            xtref.commitHash = item.sha;
+                            xtref.content = item.content;
+                            console.log(`\n   SPEC-UP-T: Match found for term: ${xtref.term} in ${xtref.externalSpec};`);
+                        } else {
+                            xtref.commitHash = "not found";
+                            xtref.content = "This term was not found in the external repository.";
+                            console.log(`\n   SPEC-UP-T: No match found for term: ${xtref.term} in ${xtref.externalSpec};`);
+                        }
+                    });
                 }
-            }        }
-    }
+            }
 
-    // Fetch all term information, then write the results to JSON and JS files.
-    fetchAllTermsInfoFromGithub(skipExisting).then(() => {
-        const allXTrefsStr = JSON.stringify(allXTrefs, null, 2);
-        fs.writeFileSync(outputPathJSON, allXTrefsStr, 'utf8');
-        const stringReadyForFileWrite = `const allXTrefs = ${allXTrefsStr};`;
-        fs.writeFileSync(outputPathJS, stringReadyForFileWrite, 'utf8');
-        fs.writeFileSync(outputPathJSTimeStamped, stringReadyForFileWrite, 'utf8');
+            const allXTrefsStr = JSON.stringify(allXTrefs, null, 2);
+            fs.writeFileSync(outputPathJSON, allXTrefsStr, 'utf8');
+            const stringReadyForFileWrite = `const allXTrefs = ${allXTrefsStr};`;
+            fs.writeFileSync(outputPathJS, stringReadyForFileWrite, 'utf8');
+            fs.writeFileSync(outputPathJSTimeStamped, stringReadyForFileWrite, 'utf8');
 
-        // Run the render function to update the HTML file
-        require('../index.js')({ nowatch: true });
-    }).catch(err => {
-        console.error('Error:', err);
-    });
+            // Run the render function to update the HTML file
+            require('../index.js')({ nowatch: true });
+        } catch (error) {
+            console.error('An error occurred:', error);
+        }
+    })();
 }
 
 module.exports = {

package/src/init.js

@@ -0,0 +1,36 @@
+require('dotenv').config();
+const fs = require('fs-extra');
+const path = require('path');
+const { exec } = require('child_process');
+const outputDir = path.join(process.cwd(), 'output');
+const initFlagPath = path.join(outputDir, 'init.flag');
+let config = fs.readJsonSync(path.join(process.cwd(), 'specs.json'));
+
+const updateXTrefs = require('./get-xtrefs-data.js').updateXTrefs;
+const { prepareTref } = require('./prepare-tref');
+
+async function initialize() {
+    try {
+        // Check if the init script has already run
+        if (await fs.pathExists(initFlagPath)) {
+            console.log('Initialization has already been completed.');
+            return;
+        }
+
+        // Place the init script here
+        
+        updateXTrefs(process.env.GITHUB_API_TOKEN, false);
+        // prepareTref(path.join(config.specs[0].spec_directory, config.specs[0].spec_terms_directory));
+
+        // End of the init script
+
+        // Create the init flag file
+        await fs.writeFile(initFlagPath, 'Initialization completed.');
+
+        console.log('Initialization complete.');
+    } catch (error) {
+        console.error(`Initialization failed: ${error.message}`);
+    }
+}
+
+module.exports = { initialize };

package/src/json-key-validator.js

@@ -34,7 +34,7 @@ function checkKeysSync(object, expectedKeys, parentKey = '') {
         } else if (typeof object === 'object') {
             // If the key is missing from the object, log an error
             if (!(key in object)) {
-                console.error(`\n   SPEC-UP-T: Error: Missing key '${key}' in ${parentKey}\n   We cannot guarantee that Spec-Up-T will work properly.\n   Here is an example specs.json file:\n   https://github.com/trustoverip/spec-up-t-starter-pack/blob/main/spec-up-t-starterpack/specs.json` + "\n");
+                console.error(`\n   SPEC-UP-T: Error: Missing key '${key}' in ${parentKey}\n   We cannot guarantee that Spec-Up-T will work properly.\n   Here is an example specs.json file:\n   https://github.com/trustoverip/spec-up-t-starter-pack/blob/main/spec-up-t-boilerplate/specs.json` + "\n");
                 errorFound = true;
                 pauseForEnterSync(); // Pause synchronously to allow user to acknowledge the error
             }

package/src/prepare-tref.js

@@ -1,3 +1,13 @@
+/**
+ * @file prepare-tref.js
+ * @description This script contains functions to process and prepare term references (trefs) from markdown files.
+ * It includes functionality to read local xtref content from a JSON file and process markdown files in a directory recursively. This script inserts the content of the local xtref (in JSON) into the markdown files. It also inserts a note that this content is temporary and can be removed.
+ * 
+ * @module prepareTref
+ * @requires fs
+ * @requires path
+ */
+
 const fs = require('fs');
 const path = require('path');
 
@@ -70,15 +80,18 @@ function prepareTref(directory) {
                                         
                                         The g flag ensures that all occurrences in the string are replaced.
                                     */
-                                    const defPart = /\[\[def: .*?\]\]:/g;
+                                    // const defPart = /\[\[def: .*?\]\]:/g;
+                                    const defPart = /\[\[def: ([^,]+),.*?\]\]/g;
+
                                     localXTrefContent.content = localXTrefContent.content.replace(defPart, '');
 
-                                    fs.writeFileSync(itemPath, match[0] + '\n\n' + '<!-- This is a copy of the saved remote text. Remove it if you like. It is automatically (re)generated --><span class="transcluded-xref-term">transcluded xref</span>' + '\n\n~ Commit Hash: ' + localXTrefContent.commitHash + localXTrefContent.content, 'utf8');
+                                    fs.writeFileSync(itemPath, match[0] + '\n\n' + '<!-- This is a copy of the saved remote text. Remove it if you like. It is automatically (re)generated --><span class="transcluded-xref-term">transcluded xref</span>' + '\n\n~ Commit Hash: ' + localXTrefContent.commitHash + '\n\n\n' + localXTrefContent.content, 'utf8');
                                 }
                             }
                         }
                     } catch (err) {
-                        console.error(`\n   SPEC-UP-T: Error reading or writing file ${item.name}: ${err}` + "\n");
+                        fs.writeFileSync(itemPath, match[0] + '\n\n' + '<!-- This is a copy of the saved remote text. Remove it if you like. It is automatically (re)generated --> Nothing found, so nothing to show.', 'utf8');
+                        // console.error(`\n   SPEC-UP-T: Error reading or writing file ${item.name}: ${err}` + "\n");
                     }
                 }
             });

package/src/utils/doesUrlExist.js

@@ -0,0 +1,15 @@
+async function doesUrlExist(repo, termsDir) {
+    const url = `${repo}/blob/main/${termsDir}`;
+    try {
+        const response = await fetch(url, { method: 'HEAD' });
+        if (response.ok) {
+            return true;
+        } else {
+            return false;
+        }
+    } catch (error) {
+        console.log('Error:', error);
+    }
+}
+
+exports.doesUrlExist = doesUrlExist;