spec-up-t 1.1.52 → 1.1.53

package/assets/js/insert-trefs.js

@@ -97,7 +97,7 @@ function insertTrefs(allXTrefs) { // Pass allXTrefs as a parameter
 
          // Clean up markdown content
          let content = xref.content
-            .replace(/\[\[def: ([^,]+),.*?\]\]/g, '') // Remove [[def: ...]]
+            .replace(/\[\[def:[^\]]*?\]\]/g, '') // Remove [[def: ...]] patterns regardless of trailing chars
             .split('\n')
             .map(line => line.replace(/^\s*~\s*/, '')) // Remove leading ~ and spaces
             .join('\n')

package/branches.md

@@ -0,0 +1,33 @@
+# Beschrijving van de branches
+
+## 
+
+
+
+
+## dev
+
+
+## master
+
+
+## test/downloadwholerepo
+
+Hierbij wordt de hele repository gedownload. Dit is een experiment om te kijken of ik de hele repository kan downloaden en dan de bestanden kan gebruiken die ik nodig heb. In plaats van de search API te gebruiken met zijn strakke limieten.
+
+
+## test/fixtref
+
+Bewaren, maar kan weg als ik uiteindelijk een andere oplossing heb gevonden.
+
+
+## test/xtrefnew
+
+Lokaal Bewaren
+Lijkt goed te werken.
+Met AI gebouwde oplossing waarbij ik probeer iets aan de API limieten te doen. Ik heb twee belangrijke bestanden weggedaan en AI ze opnieuw laten bouwen op basis van een prompt en de bestanden die ik had. Het lijkt te werken, maar ik heb geen idee of het goed is. Het is een beetje een experiment.
+
+## archive/master-pre-bootstrap
+
+
+## archive/master-pre-clientside-tref-insertion

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spec-up-t",
-  "version": "1.1.52",
+  "version": "1.1.53",
   "description": "Technical specification drafting tool that generates rich specification documents from markdown. Forked from https://github.com/decentralized-identity/spec-up by Daniel Buchner (https://github.com/csuwildcat)",
   "main": "./index",
   "repository": {
@@ -65,5 +65,11 @@
   },
   "overrides": {
     "braces": ">=3.0.3"
+  },
+  "devDependencies": {
+    "jest": "^29.7.0"
+  },
+  "scripts": {
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest"
   }
 }

package/src/README.md

@@ -0,0 +1,98 @@
+<h1>Spec-Up-T External References Collection Workflow</h1>
+<p>This codebase implements a system for collecting and processing cross-specification references from markdown files. Here's a workflow diagram of what happens when <a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/collect-external-references.js#37%2C10" data-keybinding-context="1187"><span class="icon-label">collectExternalReferences()</span></a> is called:</p>
+<h2>Workflow Overview</h2>
+<ol>
+<li>
+<p><strong>Initialization &amp; Configuration</strong></p>
+<ul>
+<li>Loads environment variables using dotenv</li>
+<li>Reads <a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/collect-external-references.js#42%2C39" data-keybinding-context="1188"><span class="icon-label">specs.json</span></a> configuration file</li>
+<li>Obtains GitHub Personal Access Token (PAT) from options or environment variables</li>
+</ul>
+</li>
+<li>
+<p><strong>Validation Checks</strong></p>
+<ul>
+<li>Verifies GitHub PAT is available, stops with instructions if missing</li>
+<li>Confirms external repositories are configured in specs.json</li>
+<li>Validates repository URLs using <a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/utils/doesUrlExist.js#8%2C16" data-keybinding-context="1189"><span class="icon-label">doesUrlExist.js</span></a></li>
+</ul>
+</li>
+<li>
+<p><strong>Content Collection</strong></p>
+<ul>
+<li>Creates necessary output directories</li>
+<li>Reads all markdown files from specified term directories</li>
+<li>Concatenates content into a single string for processing</li>
+</ul>
+</li>
+<li>
+<p><strong>Cross-Reference Extraction</strong></p>
+<ul>
+<li>Uses regex to find <code>[[xref:...]]</code> and <code>[[tref:...]]</code> patterns in markdown</li>
+<li>Processes these patterns into structured objects with <a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/collect-external-references.js#158%2C58" data-keybinding-context="1190"><span class="icon-label">externalSpec</span></a> and <a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/collect-external-references.js#186%2C64" data-keybinding-context="1191"><span class="icon-label">term</span></a> properties</li>
+<li>Filters out duplicates and keeps only references found in current markdown content</li>
+</ul>
+</li>
+<li>
+<p><strong>Metadata Enhancement</strong></p>
+<ul>
+<li>Extends each reference with additional repository information:
+<ul>
+<li>Repository URL</li>
+<li>Owner/repo names</li>
+<li>Terms directory path</li>
+<li>Avatar URL</li>
+</ul>
+</li>
+</ul>
+</li>
+<li>
+<p><strong>GitHub API Interaction</strong></p>
+<ul>
+<li>For each reference, calls <a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/collectExternalReferences/fetchTermsFromGitHubRepository.js#16%2C16" data-keybinding-context="1192"><span class="icon-label">fetchTermsFromGitHubRepository</span></a> which:
+<ul>
+<li>Uses Octokit to interface with GitHub API</li>
+<li>Implements caching to minimize API requests</li>
+<li>Handles rate limiting with throttling and retries</li>
+<li>Searches for terms within specified repositories/directories</li>
+</ul>
+</li>
+</ul>
+</li>
+<li>
+<p><strong>Term Content Processing</strong></p>
+<ul>
+<li>When terms are found, retrieves:
+<ul>
+<li>File content</li>
+<li>Latest commit hash</li>
+<li>Repository metadata</li>
+</ul>
+</li>
+<li>Uses <a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/collectExternalReferences/matchTerm.js#3%2C10" data-keybinding-context="1193"><span class="icon-label">matchTerm.js</span></a> to validate the term within content</li>
+</ul>
+</li>
+<li>
+<p><strong>Output Generation</strong></p>
+<ul>
+<li>Creates three output files:
+<ul>
+<li><code>output/xtrefs-data.json</code> - JSON data file</li>
+<li><code>output/xtrefs-data.js</code> - JavaScript module file</li>
+<li><code>output/xtrefs-history/xtrefs-data-[timestamp].js</code> - Historical record</li>
+</ul>
+</li>
+</ul>
+</li>
+</ol>
+<h2>Key Functions</h2>
+<ul>
+<li><a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/collect-external-references.js#37%2C10" data-keybinding-context="1194"><span class="icon-label">collectExternalReferences()</span></a>: Main orchestration function</li>
+<li><a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/collect-external-references.js#143%2C18" data-keybinding-context="1195"><span class="icon-label">extendXTrefs()</span></a>: Adds repository metadata to references</li>
+<li><a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/collect-external-references.js#192%2C18" data-keybinding-context="1196"><span class="icon-label">processXTref()</span></a>: Parses reference syntax into structured objects</li>
+<li><a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/collect-external-references.js#184%2C18" data-keybinding-context="1197"><span class="icon-label">isXTrefInMarkdown()</span></a>: Validates references against current content</li>
+<li><a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/collectExternalReferences/fetchTermsFromGitHubRepository.js#16%2C16" data-keybinding-context="1198"><span class="icon-label">fetchTermsFromGitHubRepository()</span></a>: Interacts with GitHub API to find terms</li>
+<li><a class="chat-inline-anchor-widget show-file-icons" title="" draggable="true" href="vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html" data-href="file:///Users/kor/webdev/Blockchain-Bird/KERI/Spec-Up/active-bcb/spec-up-t/spec-up-t/src/collectExternalReferences/processXTrefsData.js#9%2C16" data-keybinding-context="1199"><span class="icon-label">processXTrefsData()</span></a>: Processes the data and generates output files</li>
+</ul>
+<p>This system enables cross-specification linking by fetching and maintaining references to terms defined in external GitHub repositories, ensuring that documentation can link to canonical definitions maintained elsewhere.</p>

package/src/collect-external-references.js

@@ -1,16 +1,109 @@
 /**
- * @file This script is responsible for fetching the latest commit hash of term files from the GitHub API and generating both a JavaScript file and a JSON file containing the data for the cross-references (xrefs). 
+ * @file Collects and processes external reference information for cross-specification linking.
  * 
- * The generated JavaScript file is included in the HTML output of the specification, serving as a data source for the JavaScript code embedded in the HTML file. 
+ * This script fetches the latest commit hash of term files from GitHub repositories
+ * configured in specs.json, then generates both JavaScript and JSON files containing
+ * cross-reference (xref/tref) data for use in specifications.
+ * 
+ * Example of the output:
+ * 
+ * const allXTrefs = {
+    "xtrefs": [
+        {
+        "externalSpec": "toip1",
+        "term": "SSI",
+        "repoUrl": "https://github.com/henkvancann/ctwg-main-glossary",
+        "terms_dir": "spec/terms-definitions",
+        "owner": "henkvancann",
+        "repo": "ctwg-main-glossary",
+        "site": null,
+        "commitHash": "not found",
+        "content": "This term was not found in the external repository."
+        },
+        {
+        "externalSpec": "vlei1",
+        "term": "vlei-ecosystem-governance-framework",
+        "repoUrl": "https://github.com/henkvancann/vlei-glossary",
+        "terms_dir": "spec/terms-definitions",
+        "owner": "henkvancann",
+        "repo": "vlei-glossary",
+        "avatarUrl": "https://avatars.githubusercontent.com/u/479356?v=4",
+        "site": null,
+        "commitHash": "5e36b16e58984eeaccae22116a2bf058ab01a0e9",
+        "content": "[[def: vlei-ecosystem-governance-framework, vlei ecosystem governance framework]]\n\n~ The Verifiable LEI (vLEI) Ecosystem [[ref: governance-framework]] Information Trust Policies. It's a **document** that defines the … etc"
+        }
+    ]
+  };
  * 
- * Additionally, the data is written to a JSON file for further processing or usage. This ensures that the xref data is available in both JavaScript and JSON formats, providing flexibility for different use cases.
- *
  * @author Kor Dwarshuis
  * @version 1.0.0
  * @since 2024-06-09
  */
 
 
+const { shouldProcessFile } = require('./utils/file-filter');
+function isXTrefInMarkdown(xtref, markdownContent) {
+    const regex = new RegExp(`\\[\\[(?:x|t)ref:${xtref.externalSpec},\\s*${xtref.term}\\]\\]`, 'g');
+    return regex.test(markdownContent);
+}
+
+// Helper function to process an XTref string and return an object.
+function processXTref(xtref) {
+    let [externalSpec, term] = xtref
+        .replace(/\[\[(?:xref|tref):/, '')
+        .replace(/\]\]/, '')
+        .trim()
+        .split(/,/, 2);
+    return {
+        externalSpec: externalSpec.trim(),
+        term: term.trim()
+    };
+}
+
+
+// allMarkdownContent: (string) The content to search for XTrefs.
+// allXTrefs: (object) An object with an array property "xtrefs" to which new entries will be added.
+function addNewXTrefsFromMarkdown(allMarkdownContent, allXTrefs) {
+    const regex = /\[\[(?:xref|tref):.*?\]\]/g;
+    if (regex.test(allMarkdownContent)) {
+        const xtrefs = allMarkdownContent.match(regex);
+        xtrefs.forEach(xtref => {
+            const newXTrefObj = processXTref(xtref);
+            // Add newXTrefObj only if an object with the same term and externalSpec doesn't already exist.
+            if (!allXTrefs.xtrefs.some(existingXTref =>
+                existingXTref.term === newXTrefObj.term &&
+                existingXTref.externalSpec === newXTrefObj.externalSpec)) {
+                allXTrefs.xtrefs.push(newXTrefObj);
+            }
+        });
+    }
+    return allXTrefs;
+}
+
+/**
+ * Collects external references from markdown files and processes them into usable data files.
+ * 
+ * @function collectExternalReferences
+ * @param {Object} options - Configuration options
+ * @param {string} [options.pat] - GitHub Personal Access Token (overrides environment variable)
+ * @returns {void}
+ * 
+ * @description
+ * This function performs several key operations:
+ * 1. Validates GitHub PAT availability and external repository configurations
+ * 2. Checks validity of repository URLs
+ * 3. Extracts xref/tref patterns from markdown content
+ * 4. Extends references with repository metadata
+ * 5. Processes references to fetch commit information
+ * 6. Generates output files in both JS and JSON formats
+ * 
+ * @example
+ * // Basic usage
+ * collectExternalReferences();
+ * 
+ * // With explicit PAT
+ * collectExternalReferences({ pat: 'github_pat_xxxxxxxxxxxx' });
+ */
 function collectExternalReferences(options = {}) {
     require('dotenv').config();
     const fs = require('fs-extra');
@@ -157,21 +250,16 @@ function collectExternalReferences(options = {}) {
             });
         }
 
-        // Function to check if an xtref is in the markdown content
-        function isXTrefInMarkdown(xtref, markdownContent) {
-            // const regex = new RegExp(`\\[\\[xref:${xref.term}\\]\\]`, 'g');
-            const regex = new RegExp(`\\[\\[(?:x|t)ref:${xtref.term}\\]\\]`, 'g');
-            const result = regex.test(markdownContent);
-            return result;
-        }
-
         // Function to process and clean up xref / tref strings found in the markdown file, returning an object with `externalSpec` and `term` properties.
+        //TODO: check if this is correct
         function processXTref(xtref) {
             let [externalSpec, term] = xtref.replace(/\[\[(?:xref|tref):/, '').replace(/\]\]/, '').trim().split(/,/, 2);
-            return {
+            const xtrefObject = {
                 externalSpec: externalSpec.trim(),
                 term: term.trim()
             };
+
+            return xtrefObject;
         }
 
         // Initialize an object to store all xtrefs.
@@ -191,7 +279,7 @@ function collectExternalReferences(options = {}) {
         // Read all main repo Markdown files from a list of directories and concatenate their content into a single string.
         specTermsDirectories.forEach(specDirectory => {
             fs.readdirSync(specDirectory).forEach(file => {
-                if (file.endsWith('.md')) {
+                if (shouldProcessFile(file)) {
                     const markdown = fs.readFileSync(`${specDirectory}/${file}`, 'utf8');
                     allMarkdownContent += markdown;
                 }
@@ -203,21 +291,7 @@ function collectExternalReferences(options = {}) {
             return isXTrefInMarkdown(existingXTref, allMarkdownContent);
         });
 
-        // Add new entries if they are in the markdown
-        const regex = /\[\[(?:xref|tref):.*?\]\]/g;
-
-        // `regex` is the regular expression object, and `allMarkdownContent` is the string being tested. The test method returns a boolean value: true if the pattern is found within the string, and false otherwise.
-        if (regex.test(allMarkdownContent)) {
-            const xtrefs = allMarkdownContent.match(regex);
-            xtrefs.forEach(xtref => {
-                const newXTrefObj = processXTref(xtref);
-                // Ensure that newXTrefObj is only added to the xtrefs array if there isn't already an object with the same term and externalSpec properties. This helps maintain the uniqueness of entries in the array based on these two properties.
-                if (!allXTrefs.xtrefs.some(existingXTref =>
-                    existingXTref.term === newXTrefObj.term && existingXTref.externalSpec === newXTrefObj.externalSpec)) {
-                    allXTrefs.xtrefs.push(newXTrefObj);
-                }
-            });
-        };
+        addNewXTrefsFromMarkdown(allMarkdownContent, allXTrefs);
 
         // Example at this point:
         // allXTrefs.xtrefs: [
@@ -249,5 +323,8 @@ function collectExternalReferences(options = {}) {
 }
 
 module.exports = {
-    collectExternalReferences
-}
+    collectExternalReferences,
+    isXTrefInMarkdown,
+    addNewXTrefsFromMarkdown,
+    processXTref
+};

package/src/collect-external-references.test.js

@@ -0,0 +1,152 @@
+describe('isXTrefInMarkdown function', () => {
+    const isXTrefInMarkdown = require('./collect-external-references').isXTrefInMarkdown;
+
+    const testCases = [
+        {
+            name: 'basic xref format',
+            xtref: { externalSpec: 'kmg-1', term: 'authentic-chained-data-container' },
+            markdown: '[[xref:kmg-1,authentic-chained-data-container]]',
+            shouldMatch: true
+        },
+        {
+            name: 'basic tref format',
+            xtref: { externalSpec: 'kmg-1', term: 'authentic-chained-data-container' },
+            markdown: '[[tref:kmg-1,authentic-chained-data-container]]',
+            shouldMatch: true
+        },
+        {
+            name: 'xref with spaces',
+            xtref: { externalSpec: 'kmg-1', term: 'authentic-chained-data-container' },
+            markdown: '[[xref:kmg-1, authentic-chained-data-container]]',
+            shouldMatch: true
+        },
+        {
+            name: 'multiple references in content',
+            xtref: { externalSpec: 'kmg-1', term: 'authentic-chained-data-container' },
+            markdown: 'Text before [[xref:kmg-1,authentic-chained-data-container]] and text after',
+            shouldMatch: true
+        },
+        {
+            name: 'non-matching external spec',
+            xtref: { externalSpec: 'kmg-1', term: 'authentic-chained-data-container' },
+            markdown: '[[xref:different-spec,authentic-chained-data-container]]',
+            shouldMatch: false
+        },
+        {
+            name: 'non-matching term',
+            xtref: { externalSpec: 'kmg-1', term: 'authentic-chained-data-container' },
+            markdown: '[[xref:kmg-1,different-term]]',
+            shouldMatch: false
+        },
+        {
+            name: 'complex term with hyphens',
+            xtref: { externalSpec: 'vlei-1', term: 'legal-entity-identifier' },
+            markdown: '[[xref:vlei-1,legal-entity-identifier]]',
+            shouldMatch: true
+        },
+
+        // Test cases for case sensitivity
+        {
+            name: 'uppercase in externalSpec',
+            xtref: { externalSpec: 'kmg-1', term: 'authentic-chained-data-container' },
+            markdown: '[[xref:KMG-1,authentic-chained-data-container]]',
+            shouldMatch: false
+        },
+        {
+            name: 'uppercase in term',
+            xtref: { externalSpec: 'kmg-1', term: 'authentic-chained-data-container' },
+            markdown: '[[xref:kmg-1,Authentic-Chained-Data-Container]]',
+            shouldMatch: false
+        },
+        {
+            name: 'exact case match with mixed case',
+            xtref: { externalSpec: 'Mixed-Case', term: 'Some-Mixed-Case-Term' },
+            markdown: '[[xref:Mixed-Case,Some-Mixed-Case-Term]]',
+            shouldMatch: true
+        },
+
+        /*
+        The test case below verifies that:
+
+        - References can be found in a complex, multi-line document
+        - The function correctly matches when the reference appears in different parts of the document
+        - Both xref and tref formats are properly detected
+        - Line breaks and markdown formatting don't interfere with the matching
+        
+        */
+        {
+            name: 'multi-line markdown document',
+            xtref: { externalSpec: 'kmg-1', term: 'authentic-chained-data-container' },
+            markdown: `# Document Title
+
+This is a paragraph that talks about various concepts.
+
+## First Section
+Here we discuss the [[xref:kmg-1,authentic-chained-data-container]] concept.
+
+## Second Section
+This section refers to a different term.
+
+## Third Section
+And here we reference it again using [[tref:kmg-1,authentic-chained-data-container]].
+
+### Conclusion
+That's all about these references.`,
+            shouldMatch: true
+        }
+    ];
+
+    testCases.forEach(testCase => {
+        test(`${testCase.shouldMatch ? 'matches' : 'does not match'} ${testCase.name}`, () => {
+            expect(isXTrefInMarkdown(testCase.xtref, testCase.markdown)).toBe(testCase.shouldMatch);
+        });
+    });
+});
+
+
+describe('addNewXTrefsFromMarkdown', () => {
+    const addNewXTrefsFromMarkdown = require('./collect-external-references').addNewXTrefsFromMarkdown;
+
+    it('should add a new xtref from markdown content', () => {
+        const markdownContent = "Some text [[xref:specA, termA]] more text";
+        const allXTrefs = { xtrefs: [] };
+        const updatedXTrefs = addNewXTrefsFromMarkdown(markdownContent, allXTrefs);
+
+        expect(updatedXTrefs.xtrefs.length).toBe(1);
+        expect(updatedXTrefs.xtrefs[0]).toEqual({
+            externalSpec: 'specA',
+            term: 'termA'
+        });
+    });
+
+    it('should not add duplicate xtrefs', () => {
+        const markdownContent = "Content [[xref:specA, termA]] and again [[xref:specA, termA]]";
+        const allXTrefs = { xtrefs: [] };
+        const updatedXTrefs = addNewXTrefsFromMarkdown(markdownContent, allXTrefs);
+
+        expect(updatedXTrefs.xtrefs.length).toBe(1);
+    });
+
+    it('should add multiple distinct xtrefs', () => {
+        const markdownContent = "[[xref:specA, termA]] some text [[tref:specB, termB]]";
+        const allXTrefs = { xtrefs: [] };
+        const updatedXTrefs = addNewXTrefsFromMarkdown(markdownContent, allXTrefs);
+
+        expect(updatedXTrefs.xtrefs.length).toBe(2);
+        expect(updatedXTrefs.xtrefs).toEqual(
+            expect.arrayContaining([
+                { externalSpec: 'specA', term: 'termA' },
+                { externalSpec: 'specB', term: 'termB' }
+            ])
+        );
+    });
+
+    it('should not change xtrefs when no xtrefs are found in markdown content', () => {
+        const markdownContent = "This is markdown without any reference links.";
+        const initialXTrefs = { xtrefs: [] };
+        const updatedXTrefs = addNewXTrefsFromMarkdown(markdownContent, initialXTrefs);
+
+        expect(updatedXTrefs.xtrefs.length).toBe(0);
+    });
+
+});

package/src/collectExternalReferences/fetchTermsFromGitHubRepository.js

@@ -3,6 +3,8 @@ const path = require('path');
 const crypto = require('crypto'); // For generating cache keys
 const isLineWithDefinition = require('../utils/isLineWithDefinition').isLineWithDefinition;
 const { addPath, getPath, getAllPaths } = require('../config/paths');
+const { getSearchClient, getContentClient } = require('./octokitClient');
+
 
 // Directory to store cached files
 const CACHE_DIR = getPath('githubcache');
@@ -13,36 +15,21 @@ function generateCacheKey(...args) {
     return hash;
 }
 
-async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
-    const { Octokit } = await import("octokit");
-    const { throttling } = await import("@octokit/plugin-throttling");
-
-    // Create a throttled Octokit instance
-    const ThrottledOctokit = Octokit.plugin(throttling);
-    const octokit = new ThrottledOctokit({
-        auth: GITHUB_API_TOKEN,
-        throttle: {
-            onRateLimit: (retryAfter, options) => {
-                console.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
-                if (options.request.retryCount <= 1) {
-                    console.log(`Retrying after ${retryAfter} seconds...`);
-                    return true;
-                }
-            },
-            onAbuseLimit: (retryAfter, options) => {
-                console.warn(`Abuse detected for request ${options.method} ${options.url}`);
-            },
-            onSecondaryRateLimit: (retryAfter, options) => {
-                console.warn(`Secondary rate limit hit for request ${options.method} ${options.url}`);
-                if (options.request.retryCount <= 1) {
-                    console.log(`Retrying after ${retryAfter} seconds...`);
-                    return true;
-                }
-            },
-        },
-    });
 
+
+
+
+
+
+
+async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
+    console.log(`Searching for '${searchString}' in ${owner}/${repo}/${subdirectory}`);
     try {
+        // Create directory if it doesn't exist
+        if (!fs.existsSync(CACHE_DIR)) {
+            fs.mkdirSync(CACHE_DIR, { recursive: true });
+        }
+
         // Generate a cache key for the search query
         const searchCacheKey = generateCacheKey('search', searchString, owner, repo, subdirectory);
         const searchCacheFilePath = path.join(CACHE_DIR, `${searchCacheKey}.json`);
@@ -54,21 +41,22 @@ async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, ow
             console.log(`Serving search results from cache: ${searchCacheFilePath}`);
             searchResponse = JSON.parse(fs.readFileSync(searchCacheFilePath, 'utf-8'));
         } else {
-            // Perform the search using Octokit with exact match
+            // Get the search client
             console.log(`Performing search and caching results: ${searchCacheFilePath}`);
-            searchResponse = await octokit.rest.search.code({
-                q: `"${searchString}" repo:${owner}/${repo} path:${subdirectory}`, // Exact match in subdirectory
-                headers: {
-                    Accept: "application/vnd.github.v3.text-match+json", // Include text-match media type
-                },
-            });
+            const searchClient = await getSearchClient(GITHUB_API_TOKEN);
+
+            // Perform the search
+            searchResponse = await searchClient.search(searchString, owner, repo, subdirectory);
 
             // Cache the search response
             fs.writeFileSync(searchCacheFilePath, JSON.stringify(searchResponse), 'utf-8');
         }
-
-        // Log the search results
-        console.log(`Total matches for ${searchString} :`, searchResponse.data.total_count);
+        // After search
+        console.log(`Search found ${searchResponse.data.total_count} results`);
+        if (searchResponse.data.total_count === 0) {
+            console.log("No matches found - check if term exists in repository");
+            return null;
+        }
 
         /*
         
@@ -195,11 +183,12 @@ async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, ow
                             // Fetch file content from GitHub
                             console.log(`Downloading and caching file: ${fileCacheFilePath}`);
                             try {
-                                const fileContentResponse = await octokit.rest.repos.getContent({
-                                    owner: item.repository.owner.login, // Repository owner
-                                    repo: item.repository.name, // Repository name
-                                    path: item.path, // File path
-                                });
+                                const contentClient = await getContentClient(GITHUB_API_TOKEN);
+                                const fileContentResponse = await contentClient.getContent(
+                                    item.repository.owner.login,
+                                    item.repository.name,
+                                    item.path
+                                );
 
                                 // Decode the file content (it's base64-encoded)
                                 if (fileContentResponse.data.content) {
@@ -227,7 +216,13 @@ async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, ow
             }
         }
     } catch (error) {
-        console.error("Error searching GitHub or fetching file content:", error);
+        console.error("Error details:", {
+            message: error.message,
+            status: error.status,
+            type: error.constructor.name,
+            details: error.response?.data?.message || "No additional details"
+        });
+        return null;
     }
 
     // If no item is found, return null or undefined