spec-up-t 1.0.88 → 1.0.89

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spec-up-t",
-  "version": "1.0.88",
+  "version": "1.0.89",
   "description": "Technical specification drafting tool that generates rich specification documents from markdown. Forked from https://github.com/decentralized-identity/spec-up by Daniel Buchner (https://github.com/csuwildcat)",
   "main": "./index",
   "repository": {
@@ -23,9 +23,11 @@
   },
   "homepage": "https://github.com/trustoverip/spec-up-t#readme",
   "dependencies": {
+    "@octokit/plugin-throttling": "^9.4.0",
     "@traptitech/markdown-it-katex": "^3.3.0",
     "axios": "^1.7.7",
     "diff": "^7.0.0",
+    "dotenv": "^16.4.7",
     "find-pkg-dir": "^2.0.0",
     "fs-extra": "^11.2.0",
     "gulp": "4.0.2",
@@ -52,6 +54,7 @@
     "markdown-it-textual-uml": "^0.1.3",
     "markdown-it-toc-and-anchor": "^4.2.0",
     "merge-stream": "^2.0.0",
+    "octokit": "^4.1.0",
     "pdf-lib": "^1.17.1",
     "pkg-dir": "^8.0.0",
     "prismjs": "^1.24.0",

package/src/config/paths.js

@@ -0,0 +1,46 @@
+const path = require('path');
+const paths = {};
+
+// A path can be a directory or a file
+//
+// How to require:
+// const { addPath, getPath, getAllPaths } = require('./config/paths');
+//
+// // Example usage
+// addPath('exampleDir', '/path/to/examplePath');
+// console.log(getPath('examplePath'));
+// console.log(getAllPaths());
+
+// Add paths
+paths['output-local-terms'] = path.resolve('output', 'local-terms-dir');
+paths['specsjson'] = path.resolve('specs.json');
+
+module.exports = {
+    /**
+     * Adds a directory or file to the directories object.
+     * @param {string} name - The name of the directory.
+     * @param {string} dir - The path of the directory.
+     */
+    addPath: (name, dir) => {
+        paths[name] = path.resolve(dir);
+    },
+    /**
+     * Gets the resolved path of a directory or file by its name.
+     * @param {string} name - The name of the directory.
+     * @returns {string|null} The resolved path of the directory or null if not found.
+     */
+    getPath: (name) => {
+        return paths[name] ? path.resolve(paths[name]) : null;
+    },
+    /**
+     * Gets all directories or files with their resolved paths.
+     * @returns {Object} An object containing all directories with their resolved paths.
+     */
+    getAllPaths: () => {
+        const resolvedPaths = {};
+        for (const [name, dir] of Object.entries(paths)) {
+            resolvedPaths[name] = path.resolve(dir);
+        }
+        return resolvedPaths;
+    },
+};

package/src/create-versions-index.js

@@ -19,7 +19,7 @@ function createVersionsIndex(outputPath) {
     // Directory containing the version files
     const versionsDir = path.join(outputPath, 'versions');
 
-    // Check if the directory exists, if not create it
+    // Check if the directory that holds the versions / snapshots exists, if not create it
     if (!fs.existsSync(versionsDir)) {
         fs.mkdirSync(versionsDir, { recursive: true });
         console.log('Directory created:', versionsDir);

package/src/fix-markdown-files.js

@@ -75,10 +75,9 @@ function fixMarkdownFiles(directory) {
                         // Write the modified content back to the file synchronously if there were any changes
                         if (modified) {
                             fs.writeFileSync(itemPath, data, 'utf8');
-                            console.log(`\n   SPEC-UP-T: Modified ${item.name}` + "\n");
                         }
                     } catch (err) {
-                        console.error(`\n   SPEC-UP-T: Error reading or writing file ${item.name}: ${err}` + "\n");
+                        console.error(`\n   SPEC-UP-T: Error while trying to fix the markdown in file ${item.name}: ${err}` + "\n");
                     }
                 }
             });

package/src/get-xtrefs-data/checkRateLimit.js

@@ -0,0 +1,17 @@
+// Function to check the rate limit of the GitHub API
+function checkRateLimit(response) {
+    const remaining = response.headers.get('X-RateLimit-Remaining');
+    const reset = response.headers.get('X-RateLimit-Reset');
+
+    if (response.status === 403 && remaining === '0') {
+        const resetTime = new Date(reset * 1000);
+        console.error(`\n   SPEC-UP-T: Github API rate limit exceeded. Try again after ${resetTime}. See https://trustoverip.github.io/spec-up-t-website/docs/github-token/ for more info.\n`);
+        return true;
+    } else if (remaining !== null) {
+        console.log(`\n   SPEC-UP-T: Github API rate limit: ${remaining} requests remaining. See https://trustoverip.github.io/spec-up-t-website/docs/github-token/ for more info.\n`);
+    } else {
+        console.warn(`\n   SPEC-UP-T: Unable to determine rate limit status. Check your GitHub API token and network connection.\n`);
+    }
+    return false;
+}
+exports.checkRateLimit = checkRateLimit;

package/src/get-xtrefs-data/matchTerm.1.js

@@ -0,0 +1,23 @@
+function matchTerm(text, term) {
+    if (text && typeof text === 'string') {
+        const firstLine = text.split('\n')[0].trim();
+
+        // Check if the string starts with `[[def:` and ends with `]]`
+        if (!firstLine.startsWith('[[def:') || !firstLine.endsWith(']]')) {
+            console.log('String does not start with `[[def:` or end with `]]`');
+            return false;
+        }
+
+        // Remove `[[def:` from the beginning and `]]` from the end
+        let relevantPart = firstLine.slice(7, -2);
+
+        // Split the string on `,` and trim the array elements
+        let termsArray = relevantPart.split(',').map(term => term.trim());
+
+        // Check if the term is in the array
+        return termsArray.includes(term);
+    } else {
+        console.error('Invalid text:', text);
+    }
+}
+exports.matchTerm = matchTerm;

package/src/get-xtrefs-data/matchTerm.js

@@ -0,0 +1,26 @@
+function matchTerm(text, term) {
+    if (!text || typeof text !== 'string') {
+        // console.error('Invalid text:', text);
+        console.log('Nothing to match');
+        return false;
+    }
+
+    const firstLine = text.split('\n')[0].trim();
+
+    // Check if the string starts with `[[def:` and ends with `]]`
+    if (!firstLine.startsWith('[[def:') || !firstLine.endsWith(']]')) {
+        console.log('String does not start with `[[def:` or end with `]]`');
+        return false;
+    }
+
+    // Remove `[[def:` from the beginning and `]]` from the end
+    let relevantPart = firstLine.slice(7, -2);
+
+    // Split the string on `,` and trim the array elements
+    let termsArray = relevantPart.split(',').map(term => term.trim());
+
+    // Check if the term is in the array
+    return termsArray.includes(term);
+}
+
+exports.matchTerm = matchTerm;

package/src/get-xtrefs-data/searchGitHubCode.1.js

@@ -0,0 +1,69 @@
+async function searchGitHubCode(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
+    const { Octokit } = await import("octokit");
+    const { throttling } = await import("@octokit/plugin-throttling");
+
+    // Create a throttled Octokit instance
+    const ThrottledOctokit = Octokit.plugin(throttling);
+    const octokit = new ThrottledOctokit({
+        auth: GITHUB_API_TOKEN,
+        throttle: {
+            onRateLimit: (retryAfter, options) => {
+                console.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+            onAbuseLimit: (retryAfter, options) => {
+                console.warn(`Abuse detected for request ${options.method} ${options.url}`);
+            },
+            onSecondaryRateLimit: (retryAfter, options) => {
+                console.warn(`Secondary rate limit hit for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+        },
+    });
+
+    try {
+        // Perform the search using Octokit with exact match
+        const searchResponse = await octokit.rest.search.code({
+            // q: `${searchString} repo:${owner}/${repo}`, // Fuzzy search
+            q: `"${searchString}" repo:${owner}/${repo} path:${subdirectory}`, // Use quotation marks for exact match
+            // q: `"${searchString}" repo:${owner}/${repo} case:true`, // DOES NOT WORK Use quotation marks for exact match. Case sensitive search
+        });
+
+        // Log the search results
+        console.log("Total results:", searchResponse.data.total_count);
+
+        // const rateLimitResponse = await octokit.rest.rateLimit.get();
+        // console.log("Rate limit:", rateLimitResponse.data);
+
+        // Fetch the content of each file
+        for (const item of searchResponse.data.items) {
+            let content = "";
+            const fileContentResponse = await octokit.rest.repos.getContent({
+                owner: item.repository.owner.login, // Repository owner
+                repo: item.repository.name, // Repository name
+                path: item.path, // File path
+            });
+
+            // Decode the file content (it's base64-encoded)
+            if (fileContentResponse.data.content) {
+                content = Buffer.from(fileContentResponse.data.content, "base64").toString("utf-8");
+            } else {
+                // If the file is larger than 1 MB, GitHub's API will return a download URL instead of the content. 
+                console.log("File is too large. Download URL:", fileContentResponse.data.download_url);
+            }
+
+            item.content = content;
+        }
+        return searchResponse;
+    } catch (error) {
+        console.error("Error searching GitHub or fetching file content:", error);
+    }
+}
+
+exports.searchGitHubCode = searchGitHubCode;

package/src/get-xtrefs-data/searchGitHubCode.2.js

@@ -0,0 +1,77 @@
+async function searchGitHubCode(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
+    const { Octokit } = await import("octokit");
+    const { throttling } = await import("@octokit/plugin-throttling");
+
+    // Create a throttled Octokit instance
+    const ThrottledOctokit = Octokit.plugin(throttling);
+    const octokit = new ThrottledOctokit({
+        auth: GITHUB_API_TOKEN,
+        throttle: {
+            onRateLimit: (retryAfter, options) => {
+                console.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+            onAbuseLimit: (retryAfter, options) => {
+                console.warn(`Abuse detected for request ${options.method} ${options.url}`);
+            },
+            onSecondaryRateLimit: (retryAfter, options) => {
+                console.warn(`Secondary rate limit hit for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+        },
+    });
+
+    try {
+        // Perform the search using Octokit with exact match
+        const searchResponse = await octokit.rest.search.code({
+            q: `"${searchString}" repo:${owner}/${repo} path:${subdirectory}`, // Exact match in subdirectory
+        });
+
+        // Log the search results
+        console.log("Total results:", searchResponse.data.total_count);
+
+        // Fetch the content of each file
+        for (const item of searchResponse.data.items) {
+            // Check if the match is in the first line using text_matches
+            const isFirstLineMatch = item.text_matches.some(match =>
+                match.fragment.split("\n")[0].includes(searchString)
+            );
+
+            if (!isFirstLineMatch) {
+                console.log(`Skipping ${item.path}: Match not in the first line.`);
+                continue; // Skip this file
+            }
+
+            // Fetch file content
+            let content = "";
+            const fileContentResponse = await octokit.rest.repos.getContent({
+                owner: item.repository.owner.login, // Repository owner
+                repo: item.repository.name, // Repository name
+                path: item.path, // File path
+            });
+
+            // Decode the file content (it's base64-encoded)
+            if (fileContentResponse.data.content) {
+                content = Buffer.from(fileContentResponse.data.content, "base64").toString("utf-8");
+            } else {
+                // If the file is larger than 1 MB, GitHub's API will return a download URL instead of the content.
+                console.log("File is too large. Download URL:", fileContentResponse.data.download_url);
+            }
+
+            // Attach the content to the item
+            item.content = content;
+        }
+
+        return searchResponse;
+    } catch (error) {
+        console.error("Error searching GitHub or fetching file content:", error);
+    }
+}
+
+exports.searchGitHubCode = searchGitHubCode;

package/src/get-xtrefs-data/searchGitHubCode.3.js

@@ -0,0 +1,85 @@
+async function searchGitHubCode(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
+    const { Octokit } = await import("octokit");
+    const { throttling } = await import("@octokit/plugin-throttling");
+
+    // Create a throttled Octokit instance
+    const ThrottledOctokit = Octokit.plugin(throttling);
+    const octokit = new ThrottledOctokit({
+        auth: GITHUB_API_TOKEN,
+        throttle: {
+            onRateLimit: (retryAfter, options) => {
+                console.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+            onAbuseLimit: (retryAfter, options) => {
+                console.warn(`Abuse detected for request ${options.method} ${options.url}`);
+            },
+            onSecondaryRateLimit: (retryAfter, options) => {
+                console.warn(`Secondary rate limit hit for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+        },
+    });
+
+    try {
+        // Perform the search using Octokit with exact match
+        const searchResponse = await octokit.rest.search.code({
+            q: `"${searchString}" repo:${owner}/${repo} path:${subdirectory}`, // Exact match in subdirectory
+            headers: {
+                Accept: "application/vnd.github.v3.text-match+json", // Include text-match media type
+            },
+        });
+
+        // Log the search results
+        console.log("Total results:", searchResponse.data.total_count);
+
+        // Fetch the content of each file
+        for (const item of searchResponse.data.items) {
+            // Check if the match is in the first line using text_matches
+            if (!item.text_matches) {
+                console.log(`Skipping ${item.path}: No text matches found.`);
+                continue;
+            }
+
+            const isFirstLineMatch = item.text_matches.some(match =>
+                match.fragment.split("\n")[0].includes(searchString)
+            );
+
+            if (!isFirstLineMatch) {
+                console.log(`Skipping ${item.path}: Match not in the first line.`);
+                continue; // Skip this file
+            }
+
+            // Fetch file content
+            let content = "";
+            const fileContentResponse = await octokit.rest.repos.getContent({
+                owner: item.repository.owner.login, // Repository owner
+                repo: item.repository.name, // Repository name
+                path: item.path, // File path
+            });
+
+            // Decode the file content (it's base64-encoded)
+            if (fileContentResponse.data.content) {
+                content = Buffer.from(fileContentResponse.data.content, "base64").toString("utf-8");
+            } else {
+                // If the file is larger than 1 MB, GitHub's API will return a download URL instead of the content.
+                console.log("File is too large. Download URL:", fileContentResponse.data.download_url);
+            }
+
+            // Attach the content to the item
+            item.content = content;
+        }
+
+        return searchResponse;
+    } catch (error) {
+        console.error("Error searching GitHub or fetching file content:", error);
+    }
+}
+
+exports.searchGitHubCode = searchGitHubCode;

package/src/get-xtrefs-data/searchGitHubCode.4.js

@@ -0,0 +1,92 @@
+async function searchGitHubCode(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
+    const { Octokit } = await import("octokit");
+    const { throttling } = await import("@octokit/plugin-throttling");
+
+    // Create a throttled Octokit instance
+    const ThrottledOctokit = Octokit.plugin(throttling);
+    const octokit = new ThrottledOctokit({
+        auth: GITHUB_API_TOKEN,
+        throttle: {
+            onRateLimit: (retryAfter, options) => {
+                console.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+            onAbuseLimit: (retryAfter, options) => {
+                console.warn(`Abuse detected for request ${options.method} ${options.url}`);
+            },
+            onSecondaryRateLimit: (retryAfter, options) => {
+                console.warn(`Secondary rate limit hit for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+        },
+    });
+
+    try {
+        // Perform the search using Octokit with exact match
+        const searchResponse = await octokit.rest.search.code({
+            q: `"${searchString}" repo:${owner}/${repo} path:${subdirectory}`, // Exact match in subdirectory
+            headers: {
+                Accept: "application/vnd.github.v3.text-match+json", // Include text-match media type
+            },
+        });
+
+        // Log the search results
+        console.log("Total results:", searchResponse.data.total_count);
+
+        // Fetch the content of each file
+        for (const item of searchResponse.data.items) {
+            // Check if text_matches exists and is not empty
+            if (!item.text_matches || item.text_matches.length === 0) {
+                console.log(`Skipping ${item.path}: No text matches found.`);
+                continue;
+            }
+
+            // Check if the match is in the first line using text_matches
+            const isFirstLineMatch = item.text_matches.some(match => {
+                if (!match.fragment) {
+                    console.log(`Skipping ${item.path}: No fragment found in text match.`);
+                    return false;
+                }
+
+                const firstLine = match.fragment.split("\n")[0];
+                return firstLine.includes(searchString);
+            });
+
+            if (!isFirstLineMatch) {
+                console.log(`Skipping ${item.path}: Match not in the first line.`);
+                continue; // Skip this file
+            }
+
+            // Fetch file content
+            let content = "";
+            const fileContentResponse = await octokit.rest.repos.getContent({
+                owner: item.repository.owner.login, // Repository owner
+                repo: item.repository.name, // Repository name
+                path: item.path, // File path
+            });
+
+            // Decode the file content (it's base64-encoded)
+            if (fileContentResponse.data.content) {
+                content = Buffer.from(fileContentResponse.data.content, "base64").toString("utf-8");
+            } else {
+                // If the file is larger than 1 MB, GitHub's API will return a download URL instead of the content.
+                console.log("File is too large. Download URL:", fileContentResponse.data.download_url);
+            }
+
+            // Attach the content to the item
+            item.content = content;
+        }
+
+        return searchResponse;
+    } catch (error) {
+        console.error("Error searching GitHub or fetching file content:", error);
+    }
+}
+
+exports.searchGitHubCode = searchGitHubCode;

package/src/get-xtrefs-data/searchGitHubCode.5.js

@@ -0,0 +1,97 @@
+async function searchGitHubCode(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
+    const { Octokit } = await import("octokit");
+    const { throttling } = await import("@octokit/plugin-throttling");
+
+    // Create a throttled Octokit instance
+    const ThrottledOctokit = Octokit.plugin(throttling);
+    const octokit = new ThrottledOctokit({
+        auth: GITHUB_API_TOKEN,
+        throttle: {
+            onRateLimit: (retryAfter, options) => {
+                console.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+            onAbuseLimit: (retryAfter, options) => {
+                console.warn(`Abuse detected for request ${options.method} ${options.url}`);
+            },
+            onSecondaryRateLimit: (retryAfter, options) => {
+                console.warn(`Secondary rate limit hit for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+        },
+    });
+
+    try {
+        // Perform the search using Octokit with exact match
+        const searchResponse = await octokit.rest.search.code({
+            q: `"${searchString}" repo:${owner}/${repo} path:${subdirectory}`, // Exact match in subdirectory
+            headers: {
+                Accept: "application/vnd.github.v3.text-match+json", // Include text-match media type
+            },
+        });
+
+        // Log the search results
+        console.log("Total results:", searchResponse.data.total_count);
+
+        // Fetch the content of each file
+        for (const item of searchResponse.data.items) {
+            // Check if text_matches exists and is not empty
+            if (!item.text_matches || item.text_matches.length === 0) {
+                console.log(`Skipping ${item.path}: No text matches found.`);
+                continue;
+            }
+
+            // Check if the match is in the first line using text_matches
+            const isFirstLineMatch = item.text_matches.some(match => {
+                if (!match.fragment) {
+                    console.log(`Skipping ${item.path}: No fragment found in text match.`);
+                    return false;
+                }
+
+                const firstLine = match.fragment.split("\n")[0];
+                return firstLine.includes(searchString);
+            });
+
+            if (!isFirstLineMatch) {
+                console.log(`Skipping ${item.path}: Match not in the first line.`);
+                continue; // Skip this file
+            }
+
+            // Fetch file content
+            let content = "";
+            try {
+                const fileContentResponse = await octokit.rest.repos.getContent({
+                    owner: item.repository.owner.login, // Repository owner
+                    repo: item.repository.name, // Repository name
+                    path: item.path, // File path
+                });
+
+                // Decode the file content (it's base64-encoded)
+                if (fileContentResponse.data.content) {
+                    content = Buffer.from(fileContentResponse.data.content, "base64").toString("utf-8");
+                } else {
+                    // If the file is larger than 1 MB, GitHub's API will return a download URL instead of the content.
+                    console.log("File is too large. Download URL:", fileContentResponse.data.download_url);
+                }
+            } catch (error) {
+                console.error(`Error fetching content for ${item.path}:`, error);
+                content = ""; // Set content to an empty string if there's an error
+            }
+
+            // Attach the content to the item
+            item.content = content;
+        }
+
+        return searchResponse;
+    } catch (error) {
+        console.error("Error searching GitHub or fetching file content:", error);
+    }
+}
+
+exports.searchGitHubCode = searchGitHubCode;