spec-up-t 1.0.91 → 1.1.0

package/src/add-remove-xref-source.js

@@ -0,0 +1,171 @@
+const fs = require('fs');
+const path = require('path');
+
+// Resolve the path to specs.json in the root directory
+const JSON_FILE = path.resolve(process.cwd(), 'specs.json');
+
+// Check if the JSON file exists
+if (!fs.existsSync(JSON_FILE)) {
+    console.error(`Error: ${JSON_FILE} does not exist.`);
+    process.exit(1);
+}
+
+// Ask the user for inputs
+const readline = require('readline');
+const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+});
+
+let mode = ""; // To track if user wants to add or remove
+
+// Function to prompt for mode
+function askMode() {
+    rl.question('Would you like to add, remove, or view entries? (add/remove/view): ', (answer) => {
+        mode = answer.trim().toLowerCase();
+        if (mode === 'add' || mode === 'a') {
+            askQuestion(0);
+        } else if (mode === 'remove' || mode === 'r') {
+            askRemoveEntry();
+        } else if (mode === 'view' || mode === 'v') {
+            showReferences();
+        } else {
+            console.log('Invalid option. Please enter add, remove, or view.');
+            askMode();
+        }
+    });
+}
+
+// Questions for adding an entry
+const questions = [
+    'Enter URL of the repository: ',
+    'Enter URL of the GitHub page: ',
+    'Enter the directory where the terms can be found: ',
+    'Enter short unique name: ',
+];
+
+let inputs = {};
+
+// Function to ask questions for adding an entry
+function askQuestion(index) {
+    if (index === questions.length) {
+        rl.close();
+        updateJSON();
+        return;
+    }
+
+    rl.question(questions[index], (answer) => {
+        switch (index) {
+            case 0:
+                inputs.url = answer;
+                break;
+            case 1:
+                inputs.gh_page = answer;
+                break;
+            case 2:
+                inputs.terms_dir = answer;
+                break;
+            case 3:
+                inputs.external_spec = answer;
+                break;
+        }
+        askQuestion(index + 1);
+    });
+}
+
+// Function to prompt for removal
+function askRemoveEntry() {
+    rl.question('Enter the short unique name of the entry you want to remove: ', (answer) => {
+        removeEntry(answer.trim());
+        rl.close();
+    });
+}
+
+// Function to show current external references
+function showReferences() {
+    const data = JSON.parse(fs.readFileSync(JSON_FILE, 'utf8'));
+    console.log('\nCurrent external references (xref):\n\n');
+
+    data.specs[0].external_specs.forEach(spec => {
+        console.log('--- External Reference: ---');
+        console.log(`Short name: ${spec.external_spec}`);
+        console.log(`GitHub Page: ${spec.gh_page}`);
+        console.log(`URL: ${spec.url}`);
+        console.log(`Terms Directory: ${spec.terms_dir}`);
+        console.log('\n');
+    });
+    rl.close();
+}
+
+// Update the JSON file by adding an entry
+function updateJSON() {
+    try {
+        const data = JSON.parse(fs.readFileSync(JSON_FILE, 'utf8'));
+
+        if (!data.specs || !Array.isArray(data.specs) || !data.specs[0].external_specs) {
+            console.error('Error: Invalid JSON structure. "specs[0].external_specs" is missing.');
+            process.exit(1);
+        }
+
+        const externalSpecs = data.specs[0].external_specs;
+
+        // Check if the external_spec already exists
+        const exists = externalSpecs.some(
+            (entry) => entry.external_spec === inputs.external_spec
+        );
+
+        if (exists) {
+            console.log(
+                `Entry with external_spec "${inputs.external_spec}" already exists. No changes made.`
+            );
+            return;
+        }
+
+        // Add the new entry if it doesn't exist
+        externalSpecs.push(inputs);
+
+        fs.writeFileSync(JSON_FILE, JSON.stringify(data, null, 2), 'utf8');
+        // console.log(`Updated ${JSON_FILE} successfully.`);
+        console.log(`Updated successfully.`);
+    } catch (error) {
+        console.error(`Error: Failed to update ${JSON_FILE}.`, error.message);
+        process.exit(1);
+    }
+}
+
+// Remove an entry based on `external_spec`
+function removeEntry(externalSpec) {
+    try {
+        const data = JSON.parse(fs.readFileSync(JSON_FILE, 'utf8'));
+
+        if (!data.specs || !Array.isArray(data.specs) || !data.specs[0].external_specs) {
+            console.error('Error: Invalid JSON structure. "specs[0].external_specs" is missing.');
+            process.exit(1);
+        }
+
+        const externalSpecs = data.specs[0].external_specs;
+
+        // Filter out the entry with the matching external_spec
+        const filteredSpecs = externalSpecs.filter(
+            (entry) => entry.external_spec !== externalSpec
+        );
+
+        if (filteredSpecs.length === externalSpecs.length) {
+            console.log(`No entry found with external_spec "${externalSpec}".`);
+            return;
+        }
+
+        // Update the JSON structure
+        data.specs[0].external_specs = filteredSpecs;
+
+        fs.writeFileSync(JSON_FILE, JSON.stringify(data, null, 2), 'utf8');
+        // console.log(`Removed entry with external_spec "${externalSpec}" successfully.`);
+        console.log(`Removed entry successfully.`);
+    } catch (error) {
+        console.error(`Error: Failed to update ${JSON_FILE}.`, error.message);
+        process.exit(1);
+    }
+}
+
+// Start by asking the mode
+askMode();

package/src/{get-xtrefs-data.js → collect-external-references.js}

@@ -11,11 +11,13 @@
  */
 
 
-function updateXTrefs(GITHUB_API_TOKEN, skipExisting) {
+function collectExternalReferences(options = {}) {
+    require('dotenv').config();
     const fs = require('fs-extra');
     const readlineSync = require('readline-sync');
     const config = fs.readJsonSync('specs.json');
     const externalSpecsRepos = config.specs[0].external_specs;
+    const GITHUB_API_TOKEN = process.env.GITHUB_API_TOKEN;
 
     const explanationPAT =
 `\n   SPEC-UP-T: No GitHub Personal Access Token (PAT) was found.
@@ -67,11 +69,8 @@ Please add external references to the specs.json file that you will find at the
     }
 
     function main() {
-        const path = require('path');
-        const { searchGitHubCode } = require('./get-xtrefs-data/searchGitHubCode.js');
-        const { matchTerm } = require('./get-xtrefs-data/matchTerm');
+        const { processXTrefsData } = require('./collectExternalReferences/processXTrefsData.js');
         const { doesUrlExist } = require('./utils/doesUrlExist.js');
-        const { addPath, getPath, getAllPaths } = require('./config/paths');
 
         // Check if the URLs for the external specs repositories are valid, and prompt the user to abort if they are not.
         externalSpecsRepos.forEach(repo => {
@@ -83,7 +82,7 @@ Please add external references to the specs.json file that you will find at the
    
    Terms directory: ${repo.terms_dir}
 
-   Please fix the external references to the specs.json file that you will find at the root of your project.
+   Please fix the external references in the specs.json file that you will find at the root of your project.
 
    Do you want to stop? (yes/no): `);
                     if (userInput.toLowerCase() === 'yes' || userInput.toLowerCase() === 'y') {
@@ -115,6 +114,7 @@ Please add external references to the specs.json file that you will find at the
         const outputPathJSON = 'output/xtrefs-data.json';
         const outputPathJS = 'output/xtrefs-data.js';
         const outputPathJSTimeStamped = 'output/xtrefs-history/xtrefs-data-' + Date.now() + '.js';
+
         // Function to extend xtref objects with additional information, such as repository URL and directory information.
         function extendXTrefs(config, xtrefs) {
             if (config.specs[0].external_specs_repos) {
@@ -137,6 +137,7 @@ Please add external references to the specs.json file that you will find at the
                             const urlParts = new URL(xtref.repoUrl).pathname.split('/');
                             xtref.owner = urlParts[1];
                             xtref.repo = urlParts[2];
+                            xtref.avatarUrl = repo.avatar_url;
                         }
                     });
 
@@ -178,7 +179,9 @@ Please add external references to the specs.json file that you will find at the
         // If the output JSON file exists, load its data.
         if (fs.existsSync(outputPathJSON)) {
             const existingXTrefs = fs.readJsonSync(outputPathJSON);
-            allXTrefs = existingXTrefs && existingXTrefs.xtrefs ? existingXTrefs : { xtrefs: [] };
+            if (existingXTrefs && existingXTrefs.xtrefs) {
+                allXTrefs = existingXTrefs;
+            }
         }
 
         // Collect all markdown content
@@ -236,46 +239,14 @@ Please add external references to the specs.json file that you will find at the
         //     }
         // ]
 
-        (async () => {
-            try {
-                for (let xtref of allXTrefs.xtrefs) {
-                    const fetchedData = await searchGitHubCode(GITHUB_API_TOKEN, xtref.term, xtref.owner, xtref.repo, xtref.terms_dir);
-                    if (fetchedData.data.items.length === 0) {
-                        xtref.commitHash = "not found";
-                        xtref.content = "This term was not found in the external repository.";
-                    } else {
-                        fetchedData.data.items.forEach(item => {
-                            // If the term is found according to the matchTerm function (in the first line, line should start with “[[def:), etc) add the commit hash and content to the xtref object
-                            if (matchTerm(item.content, xtref.term)) {
-                                xtref.commitHash = item.sha;
-                                xtref.content = item.content;
-                                console.log(`\n   SPEC-UP-T: Match found for term: ${xtref.term} in ${xtref.externalSpec};`);
-                            } else {
-                                xtref.commitHash = "not found";
-                                xtref.content = "This term was not found in the external repository.";
-                                console.log(`\n   SPEC-UP-T: No match found for term: ${xtref.term} in ${xtref.externalSpec};`);
-                            }
-                        });
-                    }
-                }
-
-                const allXTrefsStr = JSON.stringify(allXTrefs, null, 2);
-                fs.writeFileSync(outputPathJSON, allXTrefsStr, 'utf8');
-                const stringReadyForFileWrite = `const allXTrefs = ${allXTrefsStr};`;
-                fs.writeFileSync(outputPathJS, stringReadyForFileWrite, 'utf8');
-                fs.writeFileSync(outputPathJSTimeStamped, stringReadyForFileWrite, 'utf8');
+        
+        processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, outputPathJS, outputPathJSTimeStamped, options);
 
-                // Run the render function to update the HTML file
-                require('../index.js')({ nowatch: true });
-            } catch (error) {
-                console.error('An error occurred:', error);
-            }
-        })();
     }
 
 
 }
 
 module.exports = {
-    updateXTrefs
+    collectExternalReferences
 }

package/src/collectExternalReferences/fetchTermsFromGitHubRepository.js

@@ -0,0 +1,237 @@
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto'); // For generating cache keys
+const isLineWithDefinition = require('../utils/isLineWithDefinition').isLineWithDefinition;
+const { addPath, getPath, getAllPaths } = require('../config/paths');
+
+// Directory to store cached files
+const CACHE_DIR = getPath('githubcache');
+
+// Helper function to generate a cache key
+function generateCacheKey(...args) {
+    const hash = crypto.createHash('md5').update(args.join('-')).digest('hex');
+    return hash;
+}
+
+async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
+    const { Octokit } = await import("octokit");
+    const { throttling } = await import("@octokit/plugin-throttling");
+
+    // Create a throttled Octokit instance
+    const ThrottledOctokit = Octokit.plugin(throttling);
+    const octokit = new ThrottledOctokit({
+        auth: GITHUB_API_TOKEN,
+        throttle: {
+            onRateLimit: (retryAfter, options) => {
+                console.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+            onAbuseLimit: (retryAfter, options) => {
+                console.warn(`Abuse detected for request ${options.method} ${options.url}`);
+            },
+            onSecondaryRateLimit: (retryAfter, options) => {
+                console.warn(`Secondary rate limit hit for request ${options.method} ${options.url}`);
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds...`);
+                    return true;
+                }
+            },
+        },
+    });
+
+    try {
+        // Generate a cache key for the search query
+        const searchCacheKey = generateCacheKey('search', searchString, owner, repo, subdirectory);
+        const searchCacheFilePath = path.join(CACHE_DIR, `${searchCacheKey}.json`);
+
+        let searchResponse;
+
+        // Check if the search response is already cached
+        if (fs.existsSync(searchCacheFilePath)) {
+            console.log(`Serving search results from cache: ${searchCacheFilePath}`);
+            searchResponse = JSON.parse(fs.readFileSync(searchCacheFilePath, 'utf-8'));
+        } else {
+            // Perform the search using Octokit with exact match
+            console.log(`Performing search and caching results: ${searchCacheFilePath}`);
+            searchResponse = await octokit.rest.search.code({
+                q: `"${searchString}" repo:${owner}/${repo} path:${subdirectory}`, // Exact match in subdirectory
+                headers: {
+                    Accept: "application/vnd.github.v3.text-match+json", // Include text-match media type
+                },
+            });
+
+            // Cache the search response
+            fs.writeFileSync(searchCacheFilePath, JSON.stringify(searchResponse), 'utf-8');
+        }
+
+        // Log the search results
+        console.log(`Total matches for ${searchString} :`, searchResponse.data.total_count);
+
+        /*
+        
+        Each item is a file that contains the search string one or more times. So if a search string is found in 'attribute-based-access-control.md' and 'abac.md', both files will be returned as separate items. Each item contains “text_matches”.
+
+        - text_matches can contain multiple objects if there are multiple matches in the file.
+        - fragment is a snippet of the file content around the matched search string, not the entire file content.
+
+        In example below:
+
+        - The total_count is 2, indicating there are two files that contain the search string.
+        - Each item in the items array represents a file.
+        - The text_matches array within each item contains objects representing different matches of the search string within the file.
+        - Each fragment is a snippet of the file content around the matched search string, not the entire file content.
+
+        {
+        "total_count": 2,
+        "items": [
+            {
+            "name": "example-file1.md",
+            "path": "docs/example-file1.md",
+            "sha": "abc123",
+            "url": "https://api.github.com/repositories/123456789/contents/docs/example-file1.md",
+            "git_url": "https://api.github.com/repositories/123456789/git/blobs/abc123",
+            "html_url": "https://github.com/owner/repo/blob/main/docs/example-file1.md",
+            "repository": {
+                "id": 123456789,
+                "name": "repo",
+                "full_name": "owner/repo",
+                "owner": {
+                "login": "owner",
+                "id": 12345,
+                "avatar_url": "https://avatars.githubusercontent.com/u/12345?v=4",
+                "url": "https://api.github.com/users/owner"
+                }
+            },
+            "text_matches": [
+                {
+                "object_url": "https://api.github.com/repositories/123456789/contents/docs/example-file1.md",
+                "object_type": "FileContent",
+                "property": "content",
+                "fragment": "This is an example content with the search string.",
+                "matches": [
+                    {
+                    "text": "search string",
+                    "indices": [31, 44]
+                    }
+                ]
+                },
+                {
+                "object_url": "https://api.github.com/repositories/123456789/contents/docs/example-file1.md",
+                "object_type": "FileContent",
+                "property": "content",
+                "fragment": "Another occurrence of the search string in the file.",
+                "matches": [
+                    {
+                    "text": "search string",
+                    "indices": [25, 38]
+                    }
+                ]
+                }
+            ]
+            },
+            {
+            "name": "example-file2.md",
+            "path": "docs/example-file2.md",
+            "sha": "def456",
+            "url": "https://api.github.com/repositories/123456789/contents/docs/example-file2.md",
+            "git_url": "https://api.github.com/repositories/123456789/git/blobs/def456",
+            "html_url": "https://github.com/owner/repo/blob/main/docs/example-file2.md",
+            "repository": {
+                "id": 123456789,
+                "name": "repo",
+                "full_name": "owner/repo",
+                "owner": {
+                "login": "owner",
+                "id": 12345,
+                "avatar_url": "https://avatars.githubusercontent.com/u/12345?v=4",
+                "url": "https://api.github.com/users/owner"
+                }
+            },
+            "text_matches": [
+                {
+                "object_url": "https://api.github.com/repositories/123456789/contents/docs/example-file2.md",
+                "object_type": "FileContent",
+                "property": "content",
+                "fragment": "This file also contains the search string.",
+                "matches": [
+                    {
+                    "text": "search string",
+                    "indices": [25, 38]
+                    }
+                ]
+                }
+            ]
+            }
+        ]
+        }
+        */
+
+        for (const item of searchResponse.data.items) {
+            // Check if text_matches exists and is not empty
+            if (!item.text_matches || item.text_matches.length === 0) {
+                continue;
+            }
+
+            // Loop through each text match. Can contain multiple fragments
+            for (const match of item.text_matches) {
+                // Split the fragment into lines, lines can be empty ('')
+                const lines = match.fragment.split("\n");
+                for (const line of lines) {
+                    if (isLineWithDefinition(line)) {
+                        // Generate a unique cache key for the file
+                        const fileCacheKey = generateCacheKey('file', owner, repo, item.path);
+                        const fileCacheFilePath = path.join(CACHE_DIR, `${fileCacheKey}.txt`);
+
+                        let fileContent;
+
+                        // Check if the file is already cached
+                        if (fs.existsSync(fileCacheFilePath)) {
+                            console.log(`Serving file from cache: ${fileCacheFilePath}`);
+                            fileContent = fs.readFileSync(fileCacheFilePath, 'utf-8');
+                        } else {
+                            // Fetch file content from GitHub
+                            console.log(`Downloading and caching file: ${fileCacheFilePath}`);
+                            try {
+                                const fileContentResponse = await octokit.rest.repos.getContent({
+                                    owner: item.repository.owner.login, // Repository owner
+                                    repo: item.repository.name, // Repository name
+                                    path: item.path, // File path
+                                });
+
+                                // Decode the file content (it's base64-encoded)
+                                if (fileContentResponse.data.content) {
+                                    fileContent = Buffer.from(fileContentResponse.data.content, "base64").toString("utf-8");
+                                    // Save the file to the cache
+                                    fs.writeFileSync(fileCacheFilePath, fileContent, 'utf-8');
+                                } else {
+                                    // If the file is larger than 1 MB, GitHub's API will return a download URL instead of the content.
+                                    console.log("File is too large. Download URL:", fileContentResponse.data.download_url);
+                                    fileContent = "";
+                                }
+                            } catch (error) {
+                                console.error(`Error fetching content for ${item.path}:`, error);
+                                fileContent = ""; // Set content to an empty string if there's an error
+                            }
+                        }
+
+                        // Attach the content to the item
+                        item.content = fileContent;
+
+                        // Return the item as soon as we find the correct line
+                        return item;
+                    }
+                }
+            }
+        }
+    } catch (error) {
+        console.error("Error searching GitHub or fetching file content:", error);
+    }
+
+    // If no item is found, return null or undefined
+    return null;
+}
+
+exports.fetchTermsFromGitHubRepository = fetchTermsFromGitHubRepository;

package/src/{get-xtrefs-data → collectExternalReferences}/matchTerm.js

@@ -1,3 +1,5 @@
+const isLineWithDefinition = require('../utils/isLineWithDefinition').isLineWithDefinition;
+
 function matchTerm(text, term) {
     if (!text || typeof text !== 'string') {
         // console.error('Invalid text:', text);
@@ -7,11 +9,10 @@ function matchTerm(text, term) {
 
     const firstLine = text.split('\n')[0].trim();
 
-    // Check if the string starts with `[[def:` and ends with `]]`
-    if (!firstLine.startsWith('[[def:') || !firstLine.endsWith(']]')) {
+    if (isLineWithDefinition(firstLine) === false) { 
         console.log('String does not start with `[[def:` or end with `]]`');
         return false;
-    }
+    };
 
     // Remove `[[def:` from the beginning and `]]` from the end
     let relevantPart = firstLine.slice(7, -2);

package/src/collectExternalReferences/processXTrefsData.js

@@ -0,0 +1,53 @@
+const fs = require('fs');
+const { fetchTermsFromGitHubRepository } = require('./fetchTermsFromGitHubRepository.js');
+const { matchTerm } = require('./matchTerm.js');
+const { addPath, getPath, getAllPaths } = require('../config/paths');
+
+// Directory to store cached files
+const CACHE_DIR = getPath('githubcache');
+
+async function processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, outputPathJS, outputPathJSTimeStamped, options) {
+    try {
+
+        // Clear the cache (remove the cache directory) if the cache option is set to false
+        if (options.cache === false) {
+            if (fs.existsSync(CACHE_DIR)) {
+                fs.rmdirSync(CACHE_DIR, { recursive: true });
+            }
+        }
+        
+        // Ensure the cache directory exists, so that we can store the fetched data
+        if (!fs.existsSync(CACHE_DIR)) {
+            fs.mkdirSync(CACHE_DIR, { recursive: true });
+        }
+
+        for (let xtref of allXTrefs.xtrefs) {
+            // Go and look if the term is in the external repository and if so, get the commit hash, and other meta info plus the content of the file
+            const item = await fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, xtref.term, xtref.owner, xtref.repo, xtref.terms_dir, options);
+
+            // // Check if fetchedData.data is defined
+            if (item !== null && matchTerm(item.content, xtref.term)) {
+                xtref.commitHash = item.sha;
+                xtref.content = item.content;
+                xtref.avatarUrl = item.repository.owner.avatar_url;
+                console.log(`\n   SPEC-UP-T: Match found for term: ${xtref.term} in ${xtref.externalSpec};`);
+            } else {
+                xtref.commitHash = "not found";
+                xtref.content = "This term was not found in the external repository.";
+                console.log(`\n   SPEC-UP-T: No match found for term: ${xtref.term} in ${xtref.externalSpec};`);
+            }
+        }
+
+        const allXTrefsStr = JSON.stringify(allXTrefs, null, 2);
+        fs.writeFileSync(outputPathJSON, allXTrefsStr, 'utf8');
+        const stringReadyForFileWrite = `const allXTrefs = ${allXTrefsStr};`;
+        fs.writeFileSync(outputPathJS, stringReadyForFileWrite, 'utf8');
+        fs.writeFileSync(outputPathJSTimeStamped, stringReadyForFileWrite, 'utf8');
+
+        require('../../index.js')({ nowatch: true });
+    } catch (error) {
+        console.error("An error occurred:", error);
+    }
+}
+
+module.exports.processXTrefsData = processXTrefsData;

package/src/config/paths.js

@@ -14,6 +14,7 @@ const paths = {};
 // Add paths
 paths['output-local-terms'] = path.resolve('output', 'local-terms-dir');
 paths['specsjson'] = path.resolve('specs.json');
+paths['githubcache'] = path.resolve('output', 'github-cache');
 
 module.exports = {
     /**