spec-up-t 1.1.52 → 1.1.54

package/src/collectExternalReferences/fetchTermsFromGitHubRepository.js

@@ -3,6 +3,8 @@ const path = require('path');
 const crypto = require('crypto'); // For generating cache keys
 const isLineWithDefinition = require('../utils/isLineWithDefinition').isLineWithDefinition;
 const { addPath, getPath, getAllPaths } = require('../config/paths');
+const { getSearchClient, getContentClient } = require('./octokitClient');
+
 
 // Directory to store cached files
 const CACHE_DIR = getPath('githubcache');
@@ -13,36 +15,21 @@ function generateCacheKey(...args) {
     return hash;
 }
 
-async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
-    const { Octokit } = await import("octokit");
-    const { throttling } = await import("@octokit/plugin-throttling");
-
-    // Create a throttled Octokit instance
-    const ThrottledOctokit = Octokit.plugin(throttling);
-    const octokit = new ThrottledOctokit({
-        auth: GITHUB_API_TOKEN,
-        throttle: {
-            onRateLimit: (retryAfter, options) => {
-                console.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
-                if (options.request.retryCount <= 1) {
-                    console.log(`Retrying after ${retryAfter} seconds...`);
-                    return true;
-                }
-            },
-            onAbuseLimit: (retryAfter, options) => {
-                console.warn(`Abuse detected for request ${options.method} ${options.url}`);
-            },
-            onSecondaryRateLimit: (retryAfter, options) => {
-                console.warn(`Secondary rate limit hit for request ${options.method} ${options.url}`);
-                if (options.request.retryCount <= 1) {
-                    console.log(`Retrying after ${retryAfter} seconds...`);
-                    return true;
-                }
-            },
-        },
-    });
 
+
+
+
+
+
+
+async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
+    console.log(`Searching for '${searchString}' in ${owner}/${repo}/${subdirectory}`);
     try {
+        // Create directory if it doesn't exist
+        if (!fs.existsSync(CACHE_DIR)) {
+            fs.mkdirSync(CACHE_DIR, { recursive: true });
+        }
+
         // Generate a cache key for the search query
         const searchCacheKey = generateCacheKey('search', searchString, owner, repo, subdirectory);
         const searchCacheFilePath = path.join(CACHE_DIR, `${searchCacheKey}.json`);
@@ -54,21 +41,22 @@ async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, ow
             console.log(`Serving search results from cache: ${searchCacheFilePath}`);
             searchResponse = JSON.parse(fs.readFileSync(searchCacheFilePath, 'utf-8'));
         } else {
-            // Perform the search using Octokit with exact match
+            // Get the search client
             console.log(`Performing search and caching results: ${searchCacheFilePath}`);
-            searchResponse = await octokit.rest.search.code({
-                q: `"${searchString}" repo:${owner}/${repo} path:${subdirectory}`, // Exact match in subdirectory
-                headers: {
-                    Accept: "application/vnd.github.v3.text-match+json", // Include text-match media type
-                },
-            });
+            const searchClient = await getSearchClient(GITHUB_API_TOKEN);
+
+            // Perform the search
+            searchResponse = await searchClient.search(searchString, owner, repo, subdirectory);
 
             // Cache the search response
             fs.writeFileSync(searchCacheFilePath, JSON.stringify(searchResponse), 'utf-8');
         }
-
-        // Log the search results
-        console.log(`Total matches for ${searchString} :`, searchResponse.data.total_count);
+        // After search
+        console.log(`Search found ${searchResponse.data.total_count} results`);
+        if (searchResponse.data.total_count === 0) {
+            console.log("No matches found - check if term exists in repository");
+            return null;
+        }
 
         /*
         
@@ -195,11 +183,12 @@ async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, ow
                             // Fetch file content from GitHub
                             console.log(`Downloading and caching file: ${fileCacheFilePath}`);
                             try {
-                                const fileContentResponse = await octokit.rest.repos.getContent({
-                                    owner: item.repository.owner.login, // Repository owner
-                                    repo: item.repository.name, // Repository name
-                                    path: item.path, // File path
-                                });
+                                const contentClient = await getContentClient(GITHUB_API_TOKEN);
+                                const fileContentResponse = await contentClient.getContent(
+                                    item.repository.owner.login,
+                                    item.repository.name,
+                                    item.path
+                                );
 
                                 // Decode the file content (it's base64-encoded)
                                 if (fileContentResponse.data.content) {
@@ -227,7 +216,13 @@ async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, ow
             }
         }
     } catch (error) {
-        console.error("Error searching GitHub or fetching file content:", error);
+        console.error("Error details:", {
+            message: error.message,
+            status: error.status,
+            type: error.constructor.name,
+            details: error.response?.data?.message || "No additional details"
+        });
+        return null;
     }
 
     // If no item is found, return null or undefined

package/src/collectExternalReferences/fetchTermsFromGitHubRepository.test.js

@@ -0,0 +1,385 @@
+/*
+This test suite covers the main functionality of the fetchTermsFromGitHubRepository function including:
+
+- Retrieving cached search results when available
+- Fetching and caching new results when not in cache
+- Handling API errors gracefully
+- Processing empty search results correctly
+
+The tests use Jest's mocking capabilities to avoid actual GitHub API calls and filesystem operations, making the tests fast and reliable.
+*/
+
+const fs = require('fs');
+const path = require('path');
+
+// Mock dependencies
+jest.mock('fs');
+jest.mock('path');
+jest.mock('crypto');
+jest.mock('../utils/isLineWithDefinition');
+jest.mock('../config/paths');
+
+// Replace your static mock with a configurable one
+jest.mock('./octokitClient', () => {
+    // Create mock functions that can be reconfigured in each test
+    const mockSearchClient = {
+        search: jest.fn()
+    };
+
+    const mockContentClient = {
+        getContent: jest.fn()
+    };
+
+    return {
+        getSearchClient: jest.fn().mockResolvedValue(mockSearchClient),
+        getContentClient: jest.fn().mockResolvedValue(mockContentClient)
+    };
+});
+
+// Import the mocked module to access the mock functions
+const octokitClient = require('./octokitClient');
+
+// Import the function to test
+const { fetchTermsFromGitHubRepository } = require('./fetchTermsFromGitHubRepository');
+const { getPath } = require('../config/paths');
+
+describe('fetchTermsFromGitHubRepository', () => {
+    // Setup common variables
+    const mockToken = 'mock-github-token';
+    const mockSearchString = 'test-search';
+    const mockOwner = 'test-owner';
+    const mockRepo = 'test-repo';
+    const mockSubdirectory = 'test-dir';
+    const mockCachePath = '/mock/cache/path';
+
+    const mockSearchResponse = {
+        data: {
+            total_count: 1,
+            items: [{
+                path: 'test-file.md',
+                repository: {
+                    owner: { login: 'test-owner' },
+                    name: 'test-repo'
+                },
+                text_matches: [{
+                    fragment: '[[def: test-term]]\nContent'
+                }]
+            }]
+        }
+    };
+
+    const mockFileContentResponse = {
+        data: {
+            content: Buffer.from('[[def: test-term]]\nTest file content').toString('base64')
+        }
+    };
+
+    beforeEach(() => {
+        // Reset all mocks
+        jest.clearAllMocks();
+
+        // Setup common mock returns
+        getPath.mockReturnValue(mockCachePath);
+        path.join.mockImplementation((...args) => args.join('/'));
+
+        // Mock crypto hash
+        const mockHash = {
+            update: jest.fn().mockReturnThis(),
+            digest: jest.fn().mockReturnValue('mock-hash')
+        };
+        require('crypto').createHash.mockReturnValue(mockHash);
+
+        // Default file system mocks
+        fs.existsSync.mockReturnValue(false);
+        fs.mkdirSync.mockReturnValue(undefined);
+        fs.writeFileSync.mockReturnValue(undefined);
+        fs.readFileSync.mockReturnValue('');
+
+        // Default isLineWithDefinition behavior
+        const { isLineWithDefinition } = require('../utils/isLineWithDefinition');
+        isLineWithDefinition.mockImplementation(line => {
+            return line.includes('[[def:') && line.includes(']]');
+        });
+    });
+
+    test('should return cached search results if available', async () => {
+        // Setup cache hit
+        fs.existsSync.mockReturnValue(true);
+        fs.readFileSync.mockReturnValue(JSON.stringify(mockSearchResponse));
+
+        // Also set up file content cache
+        fs.existsSync.mockImplementation((path) => {
+            return true; // Assume all files exist in cache
+        });
+
+        // Mock content response
+        fs.readFileSync.mockImplementation((path) => {
+            if (path.includes('.json')) {
+                return JSON.stringify(mockSearchResponse);
+            } else {
+                return '[[def: test-term]]\nContent';
+            }
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(fs.existsSync).toHaveBeenCalled();
+        expect(fs.readFileSync).toHaveBeenCalled();
+        expect(result).not.toBeNull();
+        expect(result.path).toBe('test-file.md');
+    });
+
+    test('should fetch and cache search results if not cached', async () => {
+        // Set up mocks for API calls
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockResolvedValueOnce(mockSearchResponse);
+
+        const mockContentClient = await octokitClient.getContentClient();
+        mockContentClient.getContent.mockResolvedValueOnce({
+            data: {
+                content: Buffer.from('[[def: test-term]]\nContent').toString('base64')
+            }
+        });
+
+        // Mock file system for cache miss
+        fs.existsSync.mockReturnValue(false);
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(fs.writeFileSync).toHaveBeenCalled();
+        expect(result).not.toBeNull();
+        expect(result.path).toBe('test-file.md');
+        expect(result.content).toBeDefined();
+    });
+
+    test('should handle API errors gracefully', async () => {
+        // Set up mock to throw an error
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockRejectedValueOnce(new Error('API error'));
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(result).toBeNull();
+    });
+
+    test('should handle empty search results', async () => {
+        // Set up mock for empty results
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockResolvedValueOnce({
+            data: {
+                total_count: 0,
+                items: []
+            }
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(result).toBeNull();
+    });
+
+    test('should handle invalid GitHub tokens', async () => {
+        // Set up mock for authentication error
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockRejectedValueOnce({
+            status: 401,
+            message: 'Bad credentials'
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            'invalid-token', mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(result).toBeNull();
+    });
+
+    test('should handle rate limiting errors', async () => {
+        // Set up mock for rate limit error
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockRejectedValueOnce({
+            status: 403,
+            message: 'API rate limit exceeded',
+            headers: {
+                'x-ratelimit-reset': (Date.now() / 1000 + 60).toString()
+            }
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(result).toBeNull();
+    });
+
+    test('should handle large files that provide download URLs instead of content', async () => {
+        // Set up search response with large file
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockResolvedValueOnce({
+            data: {
+                total_count: 1,
+                items: [{
+                    path: 'large-file.md',
+                    repository: {
+                        owner: { login: mockOwner },
+                        name: mockRepo
+                    },
+                    text_matches: [{
+                        fragment: '[[def: test-term]]\nContent'
+                    }]
+                }]
+            }
+        });
+
+        // Set up content client for large file (no content)
+        const mockContentClient = await octokitClient.getContentClient();
+        mockContentClient.getContent.mockResolvedValueOnce({
+            data: {
+                content: null,
+                download_url: 'https://raw.githubusercontent.com/test-owner/test-repo/main/large-file.md'
+            }
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(result).not.toBeNull();
+        expect(result.path).toBe('large-file.md');
+        expect(result.content).toBe("");
+    });
+
+    test('should handle files with multiple definition lines', async () => {
+        // Set up search response with file containing multiple definitions
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockResolvedValueOnce({
+            data: {
+                total_count: 1,
+                items: [{
+                    path: 'multi-def.md',
+                    repository: {
+                        owner: { login: mockOwner },
+                        name: mockRepo
+                    },
+                    text_matches: [{
+                        fragment: '[[def: term1]]\nContent'
+                    }]
+                }]
+            }
+        });
+
+        // File content with multiple definitions
+        const fileContent = `
+        [[def: term1, alias1]]\n
+        This is definition 1\n
+        \n
+        [[def: term2, alias2]]\n
+        This is definition 2
+        `;
+
+        // Set up content client for multi-def file
+        const mockContentClient = await octokitClient.getContentClient();
+        mockContentClient.getContent.mockResolvedValueOnce({
+            data: {
+                content: Buffer.from(fileContent).toString('base64')
+            }
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        expect(result).not.toBeNull();
+        expect(result.path).toBe('multi-def.md');
+        expect(result.content).toContain('term1');
+        expect(result.content).toContain('term2');
+    });
+
+    test('should handle file content fetch errors', async () => {
+        // Set up search that succeeds
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockResolvedValueOnce({
+            data: {
+                total_count: 1,
+                items: [{
+                    path: 'error-file.md',
+                    repository: {
+                        owner: { login: mockOwner },
+                        name: mockRepo
+                    },
+                    text_matches: [{
+                        fragment: '[[def: test-term]]\nContent'
+                    }]
+                }]
+            }
+        });
+
+        // Set up content fetch that fails
+        const mockContentClient = await octokitClient.getContentClient();
+        mockContentClient.getContent.mockRejectedValueOnce(new Error('File not found'));
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
+        );
+
+        // This might be null or might return empty content based on implementation
+        if (result) {
+            expect(result.content).toBe("");
+        } else {
+            expect(result).toBeNull();
+        }
+    });
+
+    test('should correctly filter by subdirectory', async () => {
+        // Set up search with multiple results in different directories
+        const mockSearchClient = await octokitClient.getSearchClient();
+        mockSearchClient.search.mockResolvedValueOnce({
+            data: {
+                total_count: 2,
+                items: [
+                    {
+                        path: 'test-dir/file1.md',
+                        repository: {
+                            owner: { login: mockOwner },
+                            name: mockRepo
+                        },
+                        text_matches: [{
+                            fragment: '[[def: term1]]\nContent'
+                        }]
+                    },
+                    {
+                        path: 'other-dir/file2.md',
+                        repository: {
+                            owner: { login: mockOwner },
+                            name: mockRepo
+                        },
+                        text_matches: [{
+                            fragment: '[[def: term2]]\nContent'
+                        }]
+                    }
+                ]
+            }
+        });
+
+        // Set up content client
+        const mockContentClient = await octokitClient.getContentClient();
+        mockContentClient.getContent.mockResolvedValueOnce({
+            data: {
+                content: Buffer.from('[[def: term1]]\nContent').toString('base64')
+            }
+        });
+
+        const result = await fetchTermsFromGitHubRepository(
+            mockToken, mockSearchString, mockOwner, mockRepo, 'test-dir'
+        );
+
+        expect(result).not.toBeNull();
+        expect(result.path).toBe('test-dir/file1.md');
+    });
+});

package/src/collectExternalReferences/matchTerm.js

@@ -2,20 +2,25 @@ const isLineWithDefinition = require('../utils/isLineWithDefinition').isLineWith
 
 function matchTerm(text, term) {
     if (!text || typeof text !== 'string') {
-        // console.error('Invalid text:', text);
-        console.log('Nothing to match');
+        console.log('Nothing to match for term:', term);
         return false;
     }
 
     const firstLine = text.split('\n')[0].trim();
 
-    if (isLineWithDefinition(firstLine) === false) { 
+    if (isLineWithDefinition(firstLine) === false) {
         console.log('String does not start with `[[def:` or end with `]]`');
         return false;
     };
 
-    // Remove `[[def:` from the beginning and `]]` from the end
-    let relevantPart = firstLine.slice(7, -2);
+    // Find the closing bracket position instead of assuming it's at the end
+    const startPos = firstLine.indexOf('[[def:') + 6;
+    const endPos = firstLine.indexOf(']]');
+
+    if (startPos === -1 || endPos === -1) return false;
+
+    // Extract text between [[def: and ]]
+    let relevantPart = firstLine.substring(startPos, endPos);
 
     // Split the string on `,` and trim the array elements
     let termsArray = relevantPart.split(',').map(term => term.trim());

package/src/collectExternalReferences/matchTerm.test.js

@@ -0,0 +1,30 @@
+const { matchTerm } = require('./matchTerm');
+
+describe('matchTerm', () => {
+    test('returns true when the term is found in a correctly formatted definition', () => {
+        const text = '[[def: term1, term2]]\nSome additional text';
+        expect(matchTerm(text, 'term1')).toBe(true);
+        expect(matchTerm(text, 'term2')).toBe(true);
+    });
+
+    test('returns false when the term is not found in the definition', () => {
+        const text = '[[def: term1, term2]]\nSome additional text';
+        expect(matchTerm(text, 'term3')).toBe(false);
+    });
+
+    test('returns false when the text is null or not a string', () => {
+        expect(matchTerm(null, 'term1')).toBe(false);
+        expect(matchTerm(123, 'term1')).toBe(false);
+    });
+
+    test('returns false when the first line is not a valid definition', () => {
+        const text = 'Invalid definition line\n[[def: term1, term2]]';
+        expect(matchTerm(text, 'term1')).toBe(false);
+    });
+
+    test('handles extra spaces correctly', () => {
+        const text = '[[def:   term1  ,   term2   ]]';
+        expect(matchTerm(text, 'term1')).toBe(true);
+        expect(matchTerm(text, 'term2')).toBe(true);
+    });
+});

package/src/collectExternalReferences/octokitClient.js

@@ -0,0 +1,96 @@
+// Simple wrapper for Octokit functionality
+let octokitClient = null;
+let initPromise = null;
+
+/**
+ * Initialize the Octokit client (only happens once)
+ */
+async function initializeClient(token) {
+    if (initPromise) return initPromise;
+
+    initPromise = (async () => {
+        try {
+            const { Octokit } = await import('octokit');
+            const { throttling } = await import('@octokit/plugin-throttling');
+
+            const ThrottledOctokit = Octokit.plugin(throttling);
+            octokitClient = new ThrottledOctokit({
+                auth: token,
+                throttle: {
+                    onRateLimit: (retryAfter, options) => {
+                        console.warn(`Request quota exhausted for ${options.method} ${options.url}`);
+                        return options.request.retryCount <= 1; // retry once
+                    },
+                    onSecondaryRateLimit: (retryAfter, options) => {
+                        console.warn(`Secondary rate limit hit for ${options.method} ${options.url}`);
+                        return options.request.retryCount <= 1; // retry once
+                    },
+                }
+            });
+
+            console.log("Octokit client initialized successfully");
+            return octokitClient;
+        } catch (error) {
+            console.error("Failed to initialize Octokit:", error);
+            initPromise = null; // Reset so we can try again later
+            throw error;
+        }
+    })();
+
+    return initPromise;
+}
+
+/**
+ * Get a GitHub search client
+ */
+async function getSearchClient(token) {
+    const client = await initializeClient(token);
+    return {
+        search: async (query, owner, repo, path) => {
+            // First check if the repo is a fork
+            let isForked = false;
+            try {
+                const repoInfo = await client.rest.repos.get({
+                    owner,
+                    repo
+                });
+                isForked = repoInfo.data.fork;
+            } catch (error) {
+                console.warn(`Could not determine fork status: ${error.message}`);
+            }
+
+            // Use appropriate search parameter based on fork status
+            const forkParam = isForked ? ' fork:true' : '';
+            const searchQuery = `${query} repo:${owner}/${repo} path:${path}${forkParam}`;
+
+            console.log(`Executing GitHub search: ${searchQuery}`);
+            return client.rest.search.code({
+                q: searchQuery,
+                headers: {
+                    Accept: "application/vnd.github.v3.text-match+json",
+                },
+            });
+        }
+    };
+}
+
+/**
+ * Get a GitHub content client
+ */
+async function getContentClient(token) {
+    const client = await initializeClient(token);
+    return {
+        getContent: async (owner, repo, path) => {
+            return client.rest.repos.getContent({
+                owner,
+                repo,
+                path
+            });
+        }
+    };
+}
+
+module.exports = {
+    getSearchClient,
+    getContentClient
+};

package/src/collectExternalReferences/processXTrefsData.js

@@ -31,10 +31,12 @@ async function processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, ou
                 xtref.content = item.content;
                 xtref.avatarUrl = item.repository.owner.avatar_url;
                 console.log(`✅ Match found for term: ${xtref.term} in ${xtref.externalSpec};`);
+                console.log("============================================\n\n");
             } else {
                 xtref.commitHash = "not found";
                 xtref.content = "This term was not found in the external repository.";
                 console.log(`ℹ️ No match found for term: ${xtref.term} in ${xtref.externalSpec};`);
+                console.log("============================================\n\n");
             }
         }