spec-up-t 1.1.51 → 1.1.53

package/src/collectExternalReferences/fetchTermsFromGitHubRepository.test.js

@@ -0,0 +1,385 @@
+/*
+This test suite covers the main functionality of the fetchTermsFromGitHubRepository function including:
+
+- Retrieving cached search results when available
+- Fetching and caching new results when not in cache
+- Handling API errors gracefully
+- Processing empty search results correctly
+
+The tests use Jest's mocking capabilities to avoid actual GitHub API calls and filesystem operations, making the tests fast and reliable.
+*/
+
+const fs = require('fs');
+const path = require('path');
+
+// Mock dependencies
+jest.mock('fs');
+jest.mock('path');
+jest.mock('crypto');
+jest.mock('../utils/isLineWithDefinition');
+jest.mock('../config/paths');
+
+// Replace your static mock with a configurable one
+jest.mock('./octokitClient', () => {
+    // Create mock functions that can be reconfigured in each test
+    const mockSearchClient = {
+        search: jest.fn()
+    };
+
+    const mockContentClient = {
+        getContent: jest.fn()
+    };
+
+    return {
+        getSearchClient: jest.fn().mockResolvedValue(mockSearchClient),
+        getContentClient: jest.fn().mockResolvedValue(mockContentClient)
+    };
+});
+
+// Import the mocked module to access the mock functions
+const octokitClient = require('./octokitClient');
+
+// Import the function to test
+const { fetchTermsFromGitHubRepository } = require('./fetchTermsFromGitHubRepository');
+const { getPath } = require('../config/paths');
+
+describe('fetchTermsFromGitHubRepository', () => {
+    // Setup common variables
+    const mockToken = 'mock-github-token';
+    const mockSearchString = 'test-search';
+    const mockOwner = 'test-owner';
+    const mockRepo = 'test-repo';
+    const mockSubdirectory = 'test-dir';
+    const mockCachePath = '/mock/cache/path';
+
+    const mockSearchResponse = {
+        data: {
+            total_count: 1,
+            items: [{
+                path: 'test-file.md',
+                repository: {
+                    owner: { login: 'test-owner' },
+                    name: 'test-repo'
+                },
+                text_matches: [{
+                    fragment: '[[def: test-term]]\nContent'
+                }]
+            }]
+        }
+    };
+
+    const mockFileContentResponse = {
+        data: {
+            content: Buffer.from('[[def: test-term]]\nTest file content').toString('base64')
+        }
+    };
+
+    beforeEach(() => {
+        // Reset all mocks
+        jest.clearAllMocks();
+
+        // Setup common mock returns
+        getPath.mockReturnValue(mockCachePath);
+        path.join.mockImplementation((...args) => args.join('/'));
+
+        // Mock crypto hash
+        const mockHash = {
+            update: jest.fn().mockReturnThis(),
+            digest: jest.fn().mockReturnValue('mock-hash')
+        };
+        require('crypto').createHash.mockReturnValue(mockHash);
+
+        // Default file system mocks
+        fs.existsSync.mockReturnValue(false);
+        fs.mkdirSync.mockReturnValue(undefined);
+        fs.writeFileSync.mockReturnValue(undefined);
+        fs.readFileSync.mockReturnValue('');
+
+        // Default isLineWithDefinition behavior
+        const { isLineWithDefinition } = require('../utils/isLineWithDefinition');
+        isLineWithDefinition.mockImplementation(line => {
+            return line.includes('[[def:') && line.includes(']]');
+        });
+    });
+
+    test('should return cached search results if available', async () => {
+        // Setup cache hit
+        fs.existsSync.mockReturnValue(true);
+        fs.readFileSync.mockReturnValue(JSON.stringify(mockSearchResponse));
+
+        // Also set up file content cache
+        fs.existsSync.mockImplementation((path) => {
+            return true; // Assume all files exist in cache
+        });
+
+        // Mock content response
+        fs.readFileSync.mockImplementation((path) => {
+            if (path.includes('.json')) {
+                return JSON.stringify(mockSearchResponse);
+            } else {
+                return '[[def: test-term]]\nContent';
+            }
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(fs.existsSync).toHaveBeenCalled();
+        expect(fs.readFileSync).toHaveBeenCalled();
+        expect(result).not.toBeNull();
+        expect(result.path).toBe('test-file.md');
+    });
+
+    test('should fetch and cache search results if not cached', async () => {
+        // Set up mocks for API calls
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockResolvedValueOnce(mockSearchResponse);
+
+        const mockContentClient = await octokitClient.getContentClient();
+        mockContentClient.getContent.mockResolvedValueOnce({
+            data: {
+                content: Buffer.from('[[def: test-term]]\nContent').toString('base64')
+            }
+        });
+
+        // Mock file system for cache miss
+        fs.existsSync.mockReturnValue(false);
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(fs.writeFileSync).toHaveBeenCalled();
+        expect(result).not.toBeNull();
+        expect(result.path).toBe('test-file.md');
+        expect(result.content).toBeDefined();
+    });
+
+    test('should handle API errors gracefully', async () => {
+        // Set up mock to throw an error
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockRejectedValueOnce(new Error('API error'));
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(result).toBeNull();
+    });
+
+    test('should handle empty search results', async () => {
+        // Set up mock for empty results
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockResolvedValueOnce({
+            data: {
+                total_count: 0,
+                items: []
+            }
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(result).toBeNull();
+    });
+
+    test('should handle invalid GitHub tokens', async () => {
+        // Set up mock for authentication error
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockRejectedValueOnce({
+            status: 401,
+            message: 'Bad credentials'
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            'invalid-token', mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(result).toBeNull();
+    });
+
+    test('should handle rate limiting errors', async () => {
+        // Set up mock for rate limit error
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockRejectedValueOnce({
+            status: 403,
+            message: 'API rate limit exceeded',
+            headers: {
+                'x-ratelimit-reset': (Date.now() / 1000 + 60).toString()
+            }
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(result).toBeNull();
+    });
+
+    test('should handle large files that provide download URLs instead of content', async () => {
+        // Set up search response with large file
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockResolvedValueOnce({
+            data: {
+                total_count: 1,
+                items: [{
+                    path: 'large-file.md',
+                    repository: {
+                        owner: { login: mockOwner },
+                        name: mockRepo
+                    },
+                    text_matches: [{
+                        fragment: '[[def: test-term]]\nContent'
+                    }]
+                }]
+            }
+        });
+
+        // Set up content client for large file (no content)
+        const mockContentClient = await octokitClient.getContentClient();
+        mockContentClient.getContent.mockResolvedValueOnce({
+            data: {
+                content: null,
+                download_url: 'https://raw.githubusercontent.com/test-owner/test-repo/main/large-file.md'
+            }
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(result).not.toBeNull();
+        expect(result.path).toBe('large-file.md');
+        expect(result.content).toBe("");
+    });
+
+    test('should handle files with multiple definition lines', async () => {
+        // Set up search response with file containing multiple definitions
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockResolvedValueOnce({
+            data: {
+                total_count: 1,
+                items: [{
+                    path: 'multi-def.md',
+                    repository: {
+                        owner: { login: mockOwner },
+                        name: mockRepo
+                    },
+                    text_matches: [{
+                        fragment: '[[def: term1]]\nContent'
+                    }]
+                }]
+            }
+        });
+
+        // File content with multiple definitions
+        const fileContent = `
+        [[def: term1, alias1]]\n
+        This is definition 1\n
+        \n
+        [[def: term2, alias2]]\n
+        This is definition 2
+        `;
+
+        // Set up content client for multi-def file
+        const mockContentClient = await octokitClient.getContentClient();
+        mockContentClient.getContent.mockResolvedValueOnce({
+            data: {
+                content: Buffer.from(fileContent).toString('base64')
+            }
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(result).not.toBeNull();
+        expect(result.path).toBe('multi-def.md');
+        expect(result.content).toContain('term1');
+        expect(result.content).toContain('term2');
+    });
+
+    test('should handle file content fetch errors', async () => {
+        // Set up search that succeeds
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockResolvedValueOnce({
+            data: {
+                total_count: 1,
+                items: [{
+                    path: 'error-file.md',
+                    repository: {
+                        owner: { login: mockOwner },
+                        name: mockRepo
+                    },
+                    text_matches: [{
+                        fragment: '[[def: test-term]]\nContent'
+                    }]
+                }]
+            }
+        });
+
+        // Set up content fetch that fails
+        const mockContentClient = await octokitClient.getContentClient();
+        mockContentClient.getContent.mockRejectedValueOnce(new Error('File not found'));
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        // This might be null or might return empty content based on implementation
+        if (result) {
+            expect(result.content).toBe("");
+        } else {
+            expect(result).toBeNull();
+        }
+    });
+
+    test('should correctly filter by subdirectory', async () => {
+        // Set up search with multiple results in different directories
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockResolvedValueOnce({
+            data: {
+                total_count: 2,
+                items: [
+                    {
+                        path: 'test-dir/file1.md',
+                        repository: {
+                            owner: { login: mockOwner },
+                            name: mockRepo
+                        },
+                        text_matches: [{
+                            fragment: '[[def: term1]]\nContent'
+                        }]
+                    },
+                    {
+                        path: 'other-dir/file2.md',
+                        repository: {
+                            owner: { login: mockOwner },
+                            name: mockRepo
+                        },
+                        text_matches: [{
+                            fragment: '[[def: term2]]\nContent'
+                        }]
+                    }
+                ]
+            }
+        });
+
+        // Set up content client
+        const mockContentClient = await octokitClient.getContentClient();
+        mockContentClient.getContent.mockResolvedValueOnce({
+            data: {
+                content: Buffer.from('[[def: term1]]\nContent').toString('base64')
+            }
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, 'test-dir'
+        );
+
+        expect(result).not.toBeNull();
+        expect(result.path).toBe('test-dir/file1.md');
+    });
+});

package/src/collectExternalReferences/matchTerm.js

@@ -2,20 +2,25 @@ const isLineWithDefinition = require('../utils/isLineWithDefinition').isLineWith
 
 function matchTerm(text, term) {
     if (!text || typeof text !== 'string') {
-        // console.error('Invalid text:', text);
-        console.log('Nothing to match');
+        console.log('Nothing to match for term:', term);
         return false;
     }
 
     const firstLine = text.split('\n')[0].trim();
 
-    if (isLineWithDefinition(firstLine) === false) { 
+    if (isLineWithDefinition(firstLine) === false) {
         console.log('String does not start with `[[def:` or end with `]]`');
         return false;
     };
 
-    // Remove `[[def:` from the beginning and `]]` from the end
-    let relevantPart = firstLine.slice(7, -2);
+    // Find the closing bracket position instead of assuming it's at the end
+    const startPos = firstLine.indexOf('[[def:') + 6;
+    const endPos = firstLine.indexOf(']]');
+
+    if (startPos === -1 || endPos === -1) return false;
+
+    // Extract text between [[def: and ]]
+    let relevantPart = firstLine.substring(startPos, endPos);
 
     // Split the string on `,` and trim the array elements
     let termsArray = relevantPart.split(',').map(term => term.trim());

package/src/collectExternalReferences/matchTerm.test.js

@@ -0,0 +1,30 @@
+const { matchTerm } = require('./matchTerm');
+
+describe('matchTerm', () => {
+    test('returns true when the term is found in a correctly formatted definition', () => {
+        const text = '[[def: term1, term2]]\nSome additional text';
+        expect(matchTerm(text, 'term1')).toBe(true);
+        expect(matchTerm(text, 'term2')).toBe(true);
+    });
+
+    test('returns false when the term is not found in the definition', () => {
+        const text = '[[def: term1, term2]]\nSome additional text';
+        expect(matchTerm(text, 'term3')).toBe(false);
+    });
+
+    test('returns false when the text is null or not a string', () => {
+        expect(matchTerm(null, 'term1')).toBe(false);
+        expect(matchTerm(123, 'term1')).toBe(false);
+    });
+
+    test('returns false when the first line is not a valid definition', () => {
+        const text = 'Invalid definition line\n[[def: term1, term2]]';
+        expect(matchTerm(text, 'term1')).toBe(false);
+    });
+
+    test('handles extra spaces correctly', () => {
+        const text = '[[def:   term1  ,   term2   ]]';
+        expect(matchTerm(text, 'term1')).toBe(true);
+        expect(matchTerm(text, 'term2')).toBe(true);
+    });
+});

package/src/collectExternalReferences/octokitClient.js

@@ -0,0 +1,96 @@
+// Simple wrapper for Octokit functionality
+let octokitClient = null;
+let initPromise = null;
+
+/**
+ * Initialize the Octokit client (only happens once)
+ */
+async function initializeClient(token) {
+    if (initPromise) return initPromise;
+
+    initPromise = (async () => {
+        try {
+            const { Octokit } = await import('octokit');
+            const { throttling } = await import('@octokit/plugin-throttling');
+
+            const ThrottledOctokit = Octokit.plugin(throttling);
+            octokitClient = new ThrottledOctokit({
+                auth: token,
+                throttle: {
+                    onRateLimit: (retryAfter, options) => {
+                        console.warn(`Request quota exhausted for ${options.method} ${options.url}`);
+                        return options.request.retryCount <= 1; // retry once
+                    },
+                    onSecondaryRateLimit: (retryAfter, options) => {
+                        console.warn(`Secondary rate limit hit for ${options.method} ${options.url}`);
+                        return options.request.retryCount <= 1; // retry once
+                    },
+                }
+            });
+
+            console.log("Octokit client initialized successfully");
+            return octokitClient;
+        } catch (error) {
+            console.error("Failed to initialize Octokit:", error);
+            initPromise = null; // Reset so we can try again later
+            throw error;
+        }
+    })();
+
+    return initPromise;
+}
+
+/**
+ * Get a GitHub search client
+ */
+async function getSearchClient(token) {
+    const client = await initializeClient(token);
+    return {
+        search: async (query, owner, repo, path) => {
+            // First check if the repo is a fork
+            let isForked = false;
+            try {
+                const repoInfo = await client.rest.repos.get({
+                    owner,
+                    repo
+                });
+                isForked = repoInfo.data.fork;
+            } catch (error) {
+                console.warn(`Could not determine fork status: ${error.message}`);
+            }
+
+            // Use appropriate search parameter based on fork status
+            const forkParam = isForked ? ' fork:true' : '';
+            const searchQuery = `${query} repo:${owner}/${repo} path:${path}${forkParam}`;
+
+            console.log(`Executing GitHub search: ${searchQuery}`);
+            return client.rest.search.code({
+                q: searchQuery,
+                headers: {
+                    Accept: "application/vnd.github.v3.text-match+json",
+                },
+            });
+        }
+    };
+}
+
+/**
+ * Get a GitHub content client
+ */
+async function getContentClient(token) {
+    const client = await initializeClient(token);
+    return {
+        getContent: async (owner, repo, path) => {
+            return client.rest.repos.getContent({
+                owner,
+                repo,
+                path
+            });
+        }
+    };
+}
+
+module.exports = {
+    getSearchClient,
+    getContentClient
+};

package/src/collectExternalReferences/processXTrefsData.js

@@ -31,10 +31,12 @@ async function processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, ou
                 xtref.content = item.content;
                 xtref.avatarUrl = item.repository.owner.avatar_url;
                 console.log(`✅ Match found for term: ${xtref.term} in ${xtref.externalSpec};`);
+                console.log("============================================\n\n");
             } else {
                 xtref.commitHash = "not found";
                 xtref.content = "This term was not found in the external repository.";
                 console.log(`ℹ️ No match found for term: ${xtref.term} in ${xtref.externalSpec};`);
+                console.log("============================================\n\n");
             }
         }
 

package/src/create-term-index.js

@@ -15,6 +15,8 @@
  * @since 2024-09-02
  */
 
+const { shouldProcessFile } = require('./utils/file-filter');
+
 function createTermIndex() {
     const fs = require('fs-extra');
     const path = require('path');
@@ -23,7 +25,7 @@ function createTermIndex() {
     const specTermDirectoryName = config.specs.map(spec => spec.spec_terms_directory);
     const outputPathJSON = path.join('output', 'term-index.json');
     const files = fs.readdirSync(path.join(specDirectories[0], specTermDirectoryName[0]))
-        .filter(file => !file.startsWith('_') && file.endsWith('.md'));
+        .filter(shouldProcessFile);
 
     const filePaths = files.map(file => specTermDirectoryName[0] + '/' + file);
 

package/src/create-term-relations.js

@@ -5,9 +5,11 @@
  * @since 2024-06-22
  */
 
+
 const fs = require('fs-extra');
 const path = require('path');
 const config = fs.readJsonSync('specs.json');
+const { shouldProcessFile } = require('./utils/file-filter');
 
 const specTermDirectoryName = config.specs.map(spec => spec.spec_directory + '/' + spec.spec_terms_directory);
 
@@ -41,7 +43,7 @@ function createTermRelations() {
         // read directory
         fs.readdirSync(specDirectory).forEach(file => {
             // read file
-            if (file.endsWith('.md')) {
+            if (shouldProcessFile(file)) {
                 const markdown = fs.readFileSync(`${specDirectory}/${file}`, 'utf8');
 
                 let regexDef = /\[\[def:.*?\]\]/g;

package/src/fix-markdown-files.js

@@ -1,5 +1,6 @@
 const fs = require('fs');
 const path = require('path');
+const { shouldProcessFile } = require('./utils/file-filter');
 
 // Function to process markdown files in a directory recursively
 function fixMarkdownFiles(directory) {
@@ -15,7 +16,7 @@ function fixMarkdownFiles(directory) {
                 if (item.isDirectory()) {
                     // If the item is a directory, call processDirectory recursively
                     processDirectory(itemPath);
-                } else if (item.isFile() && path.extname(item.name) === '.md') {
+                } else if (item.isFile() && shouldProcessFile(item.name)) {
                     try {
                         // Read the file synchronously
                         let data = fs.readFileSync(itemPath, 'utf8');

package/src/prepare-tref.js

@@ -22,6 +22,7 @@
 const fs = require('fs');
 const path = require('path');
 const dedent = require('dedent');
+const { shouldProcessFile } = require('./utils/file-filter');
 
 function getLocalXTrefContent(externalSpec, term) {
     const filePath = path.join('output', 'xtrefs-data.json');
@@ -58,7 +59,7 @@ function prepareTref(directory) {
                 if (item.isDirectory()) {
                     // If the item is a directory, call processDirectory recursively
                     processDirectory(itemPath);
-                } else if (item.isFile() && path.extname(item.name) === '.md') {
+                } else if (item.isFile() && shouldProcessFile(item.name)) {
                     try {
                         // Read the file synchronously
                         let data = fs.readFileSync(itemPath, 'utf8');

package/src/utils/file-filter.js

@@ -0,0 +1,36 @@
+/**
+ * @file Utility functions for filtering files consistently across the codebase
+ */
+
+/**
+ * Checks if a file is a Markdown file (ends with .md)
+ * @param {string} filename - The filename to check
+ * @returns {boolean} - True if the file is a Markdown file, false otherwise
+ */
+function isMarkdownFile(filename) {
+    return filename.endsWith('.md');
+}
+
+/**
+ * Checks if a file is hidden/excluded (starts with underscore)
+ * @param {string} filename - The filename to check
+ * @returns {boolean} - True if the file is hidden/excluded, false otherwise
+ */
+function isNotHiddenFile(filename) {
+    return !filename.startsWith('_');
+}
+
+/**
+ * Checks if a file should be processed (is a Markdown file and not hidden)
+ * @param {string} filename - The filename to check
+ * @returns {boolean} - True if the file should be processed, false otherwise
+ */
+function shouldProcessFile(filename) {
+    return isMarkdownFile(filename) && isNotHiddenFile(filename);
+}
+
+module.exports = {
+    isMarkdownFile,
+    isNotHiddenFile,
+    shouldProcessFile
+};

package/src/utils/isLineWithDefinition.js

@@ -1,13 +1,8 @@
-async function isLineWithDefinition(line) {
-    line = line.trim();
-    // Check if the string starts with `[[def:` and ends with `]]`
-    if (line.startsWith('[[def:') && line.endsWith(']]')) {
-        // console.log('String starts with `[[def:` and ends with `]]`');
-        return true;
-    } else {
-        // console.log('String does not start with `[[def:` or end with `]]`');
-        return false;
-    }
+function isLineWithDefinition(line) {
+    if (!line || typeof line !== 'string') return false;
+
+    // Check if the line starts with [[def: and contains ]]
+    return line.startsWith('[[def:') && line.includes(']]');
 }
 
 exports.isLineWithDefinition = isLineWithDefinition;