spec-up-t 1.1.54 → 1.2.0

package/src/collectExternalReferences/fetchTermsFromGitHubRepository.js

@@ -1,232 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-const crypto = require('crypto'); // For generating cache keys
-const isLineWithDefinition = require('../utils/isLineWithDefinition').isLineWithDefinition;
-const { addPath, getPath, getAllPaths } = require('../config/paths');
-const { getSearchClient, getContentClient } = require('./octokitClient');
-
-
-// Directory to store cached files
-const CACHE_DIR = getPath('githubcache');
-
-// Helper function to generate a cache key
-function generateCacheKey(...args) {
-    const hash = crypto.createHash('md5').update(args.join('-')).digest('hex');
-    return hash;
-}
-
-
-
-
-
-
-
-
-async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
-    console.log(`Searching for '${searchString}' in ${owner}/${repo}/${subdirectory}`);
-    try {
-        // Create directory if it doesn't exist
-        if (!fs.existsSync(CACHE_DIR)) {
-            fs.mkdirSync(CACHE_DIR, { recursive: true });
-        }
-
-        // Generate a cache key for the search query
-        const searchCacheKey = generateCacheKey('search', searchString, owner, repo, subdirectory);
-        const searchCacheFilePath = path.join(CACHE_DIR, `${searchCacheKey}.json`);
-
-        let searchResponse;
-
-        // Check if the search response is already cached
-        if (fs.existsSync(searchCacheFilePath)) {
-            console.log(`Serving search results from cache: ${searchCacheFilePath}`);
-            searchResponse = JSON.parse(fs.readFileSync(searchCacheFilePath, 'utf-8'));
-        } else {
-            // Get the search client
-            console.log(`Performing search and caching results: ${searchCacheFilePath}`);
-            const searchClient = await getSearchClient(GITHUB_API_TOKEN);
-
-            // Perform the search
-            searchResponse = await searchClient.search(searchString, owner, repo, subdirectory);
-
-            // Cache the search response
-            fs.writeFileSync(searchCacheFilePath, JSON.stringify(searchResponse), 'utf-8');
-        }
-        // After search
-        console.log(`Search found ${searchResponse.data.total_count} results`);
-        if (searchResponse.data.total_count === 0) {
-            console.log("No matches found - check if term exists in repository");
-            return null;
-        }
-
-        /*
-        
-        Each item is a file that contains the search string one or more times. So if a search string is found in 'attribute-based-access-control.md' and 'abac.md', both files will be returned as separate items. Each item contains “text_matches”.
-
-        - text_matches can contain multiple objects if there are multiple matches in the file.
-        - fragment is a snippet of the file content around the matched search string, not the entire file content.
-
-        In example below:
-
-        - The total_count is 2, indicating there are two files that contain the search string.
-        - Each item in the items array represents a file.
-        - The text_matches array within each item contains objects representing different matches of the search string within the file.
-        - Each fragment is a snippet of the file content around the matched search string, not the entire file content.
-
-        {
-        "total_count": 2,
-        "items": [
-            {
-            "name": "example-file1.md",
-            "path": "docs/example-file1.md",
-            "sha": "abc123",
-            "url": "https://api.github.com/repositories/123456789/contents/docs/example-file1.md",
-            "git_url": "https://api.github.com/repositories/123456789/git/blobs/abc123",
-            "html_url": "https://github.com/owner/repo/blob/main/docs/example-file1.md",
-            "repository": {
-                "id": 123456789,
-                "name": "repo",
-                "full_name": "owner/repo",
-                "owner": {
-                "login": "owner",
-                "id": 12345,
-                "avatar_url": "https://avatars.githubusercontent.com/u/12345?v=4",
-                "url": "https://api.github.com/users/owner"
-                }
-            },
-            "text_matches": [
-                {
-                "object_url": "https://api.github.com/repositories/123456789/contents/docs/example-file1.md",
-                "object_type": "FileContent",
-                "property": "content",
-                "fragment": "This is an example content with the search string.",
-                "matches": [
-                    {
-                    "text": "search string",
-                    "indices": [31, 44]
-                    }
-                ]
-                },
-                {
-                "object_url": "https://api.github.com/repositories/123456789/contents/docs/example-file1.md",
-                "object_type": "FileContent",
-                "property": "content",
-                "fragment": "Another occurrence of the search string in the file.",
-                "matches": [
-                    {
-                    "text": "search string",
-                    "indices": [25, 38]
-                    }
-                ]
-                }
-            ]
-            },
-            {
-            "name": "example-file2.md",
-            "path": "docs/example-file2.md",
-            "sha": "def456",
-            "url": "https://api.github.com/repositories/123456789/contents/docs/example-file2.md",
-            "git_url": "https://api.github.com/repositories/123456789/git/blobs/def456",
-            "html_url": "https://github.com/owner/repo/blob/main/docs/example-file2.md",
-            "repository": {
-                "id": 123456789,
-                "name": "repo",
-                "full_name": "owner/repo",
-                "owner": {
-                "login": "owner",
-                "id": 12345,
-                "avatar_url": "https://avatars.githubusercontent.com/u/12345?v=4",
-                "url": "https://api.github.com/users/owner"
-                }
-            },
-            "text_matches": [
-                {
-                "object_url": "https://api.github.com/repositories/123456789/contents/docs/example-file2.md",
-                "object_type": "FileContent",
-                "property": "content",
-                "fragment": "This file also contains the search string.",
-                "matches": [
-                    {
-                    "text": "search string",
-                    "indices": [25, 38]
-                    }
-                ]
-                }
-            ]
-            }
-        ]
-        }
-        */
-
-        for (const item of searchResponse.data.items) {
-            // Check if text_matches exists and is not empty
-            if (!item.text_matches || item.text_matches.length === 0) {
-                continue;
-            }
-
-            // Loop through each text match. Can contain multiple fragments
-            for (const match of item.text_matches) {
-                // Split the fragment into lines, lines can be empty ('')
-                const lines = match.fragment.split("\n");
-                for (const line of lines) {
-                    if (isLineWithDefinition(line)) {
-                        // Generate a unique cache key for the file
-                        const fileCacheKey = generateCacheKey('file', owner, repo, item.path);
-                        const fileCacheFilePath = path.join(CACHE_DIR, `${fileCacheKey}.txt`);
-
-                        let fileContent;
-
-                        // Check if the file is already cached
-                        if (fs.existsSync(fileCacheFilePath)) {
-                            console.log(`Serving file from cache: ${fileCacheFilePath}`);
-                            fileContent = fs.readFileSync(fileCacheFilePath, 'utf-8');
-                        } else {
-                            // Fetch file content from GitHub
-                            console.log(`Downloading and caching file: ${fileCacheFilePath}`);
-                            try {
-                                const contentClient = await getContentClient(GITHUB_API_TOKEN);
-                                const fileContentResponse = await contentClient.getContent(
-                                    item.repository.owner.login,
-                                    item.repository.name,
-                                    item.path
-                                );
-
-                                // Decode the file content (it's base64-encoded)
-                                if (fileContentResponse.data.content) {
-                                    fileContent = Buffer.from(fileContentResponse.data.content, "base64").toString("utf-8");
-                                    // Save the file to the cache
-                                    fs.writeFileSync(fileCacheFilePath, fileContent, 'utf-8');
-                                } else {
-                                    // If the file is larger than 1 MB, GitHub's API will return a download URL instead of the content.
-                                    console.log("File is too large. Download URL:", fileContentResponse.data.download_url);
-                                    fileContent = "";
-                                }
-                            } catch (error) {
-                                console.error(`Error fetching content for ${item.path}:`, error);
-                                fileContent = ""; // Set content to an empty string if there's an error
-                            }
-                        }
-
-                        // Attach the content to the item
-                        item.content = fileContent;
-
-                        // Return the item as soon as we find the correct line
-                        return item;
-                    }
-                }
-            }
-        }
-    } catch (error) {
-        console.error("Error details:", {
-            message: error.message,
-            status: error.status,
-            type: error.constructor.name,
-            details: error.response?.data?.message || "No additional details"
-        });
-        return null;
-    }
-
-    // If no item is found, return null or undefined
-    return null;
-}
-
-exports.fetchTermsFromGitHubRepository = fetchTermsFromGitHubRepository;

package/src/collectExternalReferences/fetchTermsFromGitHubRepository.test.js

@@ -1,385 +0,0 @@
-/*
-This test suite covers the main functionality of the fetchTermsFromGitHubRepository function including:
-
-- Retrieving cached search results when available
-- Fetching and caching new results when not in cache
-- Handling API errors gracefully
-- Processing empty search results correctly
-
-The tests use Jest's mocking capabilities to avoid actual GitHub API calls and filesystem operations, making the tests fast and reliable.
-*/
-
-const fs = require('fs');
-const path = require('path');
-
-// Mock dependencies
-jest.mock('fs');
-jest.mock('path');
-jest.mock('crypto');
-jest.mock('../utils/isLineWithDefinition');
-jest.mock('../config/paths');
-
-// Replace your static mock with a configurable one
-jest.mock('./octokitClient', () => {
-    // Create mock functions that can be reconfigured in each test
-    const mockSearchClient = {
-        search: jest.fn()
-    };
-
-    const mockContentClient = {
-        getContent: jest.fn()
-    };
-
-    return {
-        getSearchClient: jest.fn().mockResolvedValue(mockSearchClient),
-        getContentClient: jest.fn().mockResolvedValue(mockContentClient)
-    };
-});
-
-// Import the mocked module to access the mock functions
-const octokitClient = require('./octokitClient');
-
-// Import the function to test
-const { fetchTermsFromGitHubRepository } = require('./fetchTermsFromGitHubRepository');
-const { getPath } = require('../config/paths');
-
-describe('fetchTermsFromGitHubRepository', () => {
-    // Setup common variables
-    const mockToken = 'mock-github-token';
-    const mockSearchString = 'test-search';
-    const mockOwner = 'test-owner';
-    const mockRepo = 'test-repo';
-    const mockSubdirectory = 'test-dir';
-    const mockCachePath = '/mock/cache/path';
-
-    const mockSearchResponse = {
-        data: {
-            total_count: 1,
-            items: [{
-                path: 'test-file.md',
-                repository: {
-                    owner: { login: 'test-owner' },
-                    name: 'test-repo'
-                },
-                text_matches: [{
-                    fragment: '[[def: test-term]]\nContent'
-                }]
-            }]
-        }
-    };
-
-    const mockFileContentResponse = {
-        data: {
-            content: Buffer.from('[[def: test-term]]\nTest file content').toString('base64')
-        }
-    };
-
-    beforeEach(() => {
-        // Reset all mocks
-        jest.clearAllMocks();
-
-        // Setup common mock returns
-        getPath.mockReturnValue(mockCachePath);
-        path.join.mockImplementation((...args) => args.join('/'));
-
-        // Mock crypto hash
-        const mockHash = {
-            update: jest.fn().mockReturnThis(),
-            digest: jest.fn().mockReturnValue('mock-hash')
-        };
-        require('crypto').createHash.mockReturnValue(mockHash);
-
-        // Default file system mocks
-        fs.existsSync.mockReturnValue(false);
-        fs.mkdirSync.mockReturnValue(undefined);
-        fs.writeFileSync.mockReturnValue(undefined);
-        fs.readFileSync.mockReturnValue('');
-
-        // Default isLineWithDefinition behavior
-        const { isLineWithDefinition } = require('../utils/isLineWithDefinition');
-        isLineWithDefinition.mockImplementation(line => {
-            return line.includes('[[def:') && line.includes(']]');
-        });
-    });
-
-    test('should return cached search results if available', async () => {
-        // Setup cache hit
-        fs.existsSync.mockReturnValue(true);
-        fs.readFileSync.mockReturnValue(JSON.stringify(mockSearchResponse));
-
-        // Also set up file content cache
-        fs.existsSync.mockImplementation((path) => {
-            return true; // Assume all files exist in cache
-        });
-
-        // Mock content response
-        fs.readFileSync.mockImplementation((path) => {
-            if (path.includes('.json')) {
-                return JSON.stringify(mockSearchResponse);
-            } else {
-                return '[[def: test-term]]\nContent';
-            }
-        });
-
-        const result = await fetchTermsFromGitHubRepository(
-            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
-        );
-
-        expect(fs.existsSync).toHaveBeenCalled();
-        expect(fs.readFileSync).toHaveBeenCalled();
-        expect(result).not.toBeNull();
-        expect(result.path).toBe('test-file.md');
-    });
-
-    test('should fetch and cache search results if not cached', async () => {
-        // Set up mocks for API calls
-        const mockSearchClient = await octokitClient.getSearchClient();
-        mockSearchClient.search.mockResolvedValueOnce(mockSearchResponse);
-
-        const mockContentClient = await octokitClient.getContentClient();
-        mockContentClient.getContent.mockResolvedValueOnce({
-            data: {
-                content: Buffer.from('[[def: test-term]]\nContent').toString('base64')
-            }
-        });
-
-        // Mock file system for cache miss
-        fs.existsSync.mockReturnValue(false);
-
-        const result = await fetchTermsFromGitHubRepository(
-            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
-        );
-
-        expect(fs.writeFileSync).toHaveBeenCalled();
-        expect(result).not.toBeNull();
-        expect(result.path).toBe('test-file.md');
-        expect(result.content).toBeDefined();
-    });
-
-    test('should handle API errors gracefully', async () => {
-        // Set up mock to throw an error
-        const mockSearchClient = await octokitClient.getSearchClient();
-        mockSearchClient.search.mockRejectedValueOnce(new Error('API error'));
-
-        const result = await fetchTermsFromGitHubRepository(
-            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
-        );
-
-        expect(result).toBeNull();
-    });
-
-    test('should handle empty search results', async () => {
-        // Set up mock for empty results
-        const mockSearchClient = await octokitClient.getSearchClient();
-        mockSearchClient.search.mockResolvedValueOnce({
-            data: {
-                total_count: 0,
-                items: []
-            }
-        });
-
-        const result = await fetchTermsFromGitHubRepository(
-            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
-        );
-
-        expect(result).toBeNull();
-    });
-
-    test('should handle invalid GitHub tokens', async () => {
-        // Set up mock for authentication error
-        const mockSearchClient = await octokitClient.getSearchClient();
-        mockSearchClient.search.mockRejectedValueOnce({
-            status: 401,
-            message: 'Bad credentials'
-        });
-
-        const result = await fetchTermsFromGitHubRepository(
-            'invalid-token', mockSearchString, mockOwner, mockRepo, mockSubdirectory
-        );
-
-        expect(result).toBeNull();
-    });
-
-    test('should handle rate limiting errors', async () => {
-        // Set up mock for rate limit error
-        const mockSearchClient = await octokitClient.getSearchClient();
-        mockSearchClient.search.mockRejectedValueOnce({
-            status: 403,
-            message: 'API rate limit exceeded',
-            headers: {
-                'x-ratelimit-reset': (Date.now() / 1000 + 60).toString()
-            }
-        });
-
-        const result = await fetchTermsFromGitHubRepository(
-            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
-        );
-
-        expect(result).toBeNull();
-    });
-
-    test('should handle large files that provide download URLs instead of content', async () => {
-        // Set up search response with large file
-        const mockSearchClient = await octokitClient.getSearchClient();
-        mockSearchClient.search.mockResolvedValueOnce({
-            data: {
-                total_count: 1,
-                items: [{
-                    path: 'large-file.md',
-                    repository: {
-                        owner: { login: mockOwner },
-                        name: mockRepo
-                    },
-                    text_matches: [{
-                        fragment: '[[def: test-term]]\nContent'
-                    }]
-                }]
-            }
-        });
-
-        // Set up content client for large file (no content)
-        const mockContentClient = await octokitClient.getContentClient();
-        mockContentClient.getContent.mockResolvedValueOnce({
-            data: {
-                content: null,
-                download_url: 'https://raw.githubusercontent.com/test-owner/test-repo/main/large-file.md'
-            }
-        });
-
-        const result = await fetchTermsFromGitHubRepository(
-            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
-        );
-
-        expect(result).not.toBeNull();
-        expect(result.path).toBe('large-file.md');
-        expect(result.content).toBe("");
-    });
-
-    test('should handle files with multiple definition lines', async () => {
-        // Set up search response with file containing multiple definitions
-        const mockSearchClient = await octokitClient.getSearchClient();
-        mockSearchClient.search.mockResolvedValueOnce({
-            data: {
-                total_count: 1,
-                items: [{
-                    path: 'multi-def.md',
-                    repository: {
-                        owner: { login: mockOwner },
-                        name: mockRepo
-                    },
-                    text_matches: [{
-                        fragment: '[[def: term1]]\nContent'
-                    }]
-                }]
-            }
-        });
-
-        // File content with multiple definitions
-        const fileContent = `
-        [[def: term1, alias1]]\n
-        This is definition 1\n
-        \n
-        [[def: term2, alias2]]\n
-        This is definition 2
-        `;
-
-        // Set up content client for multi-def file
-        const mockContentClient = await octokitClient.getContentClient();
-        mockContentClient.getContent.mockResolvedValueOnce({
-            data: {
-                content: Buffer.from(fileContent).toString('base64')
-            }
-        });
-
-        const result = await fetchTermsFromGitHubRepository(
-            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
-        );
-
-        expect(result).not.toBeNull();
-        expect(result.path).toBe('multi-def.md');
-        expect(result.content).toContain('term1');
-        expect(result.content).toContain('term2');
-    });
-
-    test('should handle file content fetch errors', async () => {
-        // Set up search that succeeds
-        const mockSearchClient = await octokitClient.getSearchClient();
-        mockSearchClient.search.mockResolvedValueOnce({
-            data: {
-                total_count: 1,
-                items: [{
-                    path: 'error-file.md',
-                    repository: {
-                        owner: { login: mockOwner },
-                        name: mockRepo
-                    },
-                    text_matches: [{
-                        fragment: '[[def: test-term]]\nContent'
-                    }]
-                }]
-            }
-        });
-
-        // Set up content fetch that fails
-        const mockContentClient = await octokitClient.getContentClient();
-        mockContentClient.getContent.mockRejectedValueOnce(new Error('File not found'));
-
-        const result = await fetchTermsFromGitHubRepository(
-            mockToken, mockSearchString, mockOwner, mockRepo, mockSubdirectory
-        );
-
-        // This might be null or might return empty content based on implementation
-        if (result) {
-            expect(result.content).toBe("");
-        } else {
-            expect(result).toBeNull();
-        }
-    });
-
-    test('should correctly filter by subdirectory', async () => {
-        // Set up search with multiple results in different directories
-        const mockSearchClient = await octokitClient.getSearchClient();
-        mockSearchClient.search.mockResolvedValueOnce({
-            data: {
-                total_count: 2,
-                items: [
-                    {
-                        path: 'test-dir/file1.md',
-                        repository: {
-                            owner: { login: mockOwner },
-                            name: mockRepo
-                        },
-                        text_matches: [{
-                            fragment: '[[def: term1]]\nContent'
-                        }]
-                    },
-                    {
-                        path: 'other-dir/file2.md',
-                        repository: {
-                            owner: { login: mockOwner },
-                            name: mockRepo
-                        },
-                        text_matches: [{
-                            fragment: '[[def: term2]]\nContent'
-                        }]
-                    }
-                ]
-            }
-        });
-
-        // Set up content client
-        const mockContentClient = await octokitClient.getContentClient();
-        mockContentClient.getContent.mockResolvedValueOnce({
-            data: {
-                content: Buffer.from('[[def: term1]]\nContent').toString('base64')
-            }
-        });
-
-        const result = await fetchTermsFromGitHubRepository(
-            mockToken, mockSearchString, mockOwner, mockRepo, 'test-dir'
-        );
-
-        expect(result).not.toBeNull();
-        expect(result.path).toBe('test-dir/file1.md');
-    });
-});