spec-up-t 1.0.90 → 1.0.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/compiled/body.js +1 -1
- package/assets/compiled/head.css +1 -1
- package/assets/css/index.css +202 -150
- package/assets/js/show-commit-hashes.js +2 -2
- package/index.js +0 -3
- package/package.json +2 -1
- package/src/add-remove-xref-source.js +171 -0
- package/src/collect-external-references.js +252 -0
- package/src/collectExternalReferences/fetchTermsFromGitHubRepository.js +237 -0
- package/src/{get-xtrefs-data → collectExternalReferences}/matchTerm.js +4 -3
- package/src/collectExternalReferences/processXTrefsData.js +53 -0
- package/src/config/paths.js +1 -0
- package/src/configure.js +96 -0
- package/src/init.js +2 -8
- package/src/prepare-tref.js +21 -4
- package/src/utils/doesUrlExist.js +18 -10
- package/src/utils/isLineWithDefinition.js +13 -0
- package/src/get-xtrefs-data/matchTerm.1.js +0 -23
- package/src/get-xtrefs-data/searchGitHubCode.1.js +0 -69
- package/src/get-xtrefs-data/searchGitHubCode.2.js +0 -77
- package/src/get-xtrefs-data/searchGitHubCode.3.js +0 -85
- package/src/get-xtrefs-data/searchGitHubCode.4.js +0 -92
- package/src/get-xtrefs-data/searchGitHubCode.5.js +0 -97
- package/src/get-xtrefs-data/searchGitHubCode.js +0 -97
- package/src/get-xtrefs-data.js +0 -222
- /package/src/{get-xtrefs-data → collectExternalReferences}/checkRateLimit.js +0 -0
- /package/src/{get-xtrefs-data → collectExternalReferences}/setupFetchHeaders.js +0 -0
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
|
|
4
|
+
// Resolve the path to specs.json in the root directory
|
|
5
|
+
const JSON_FILE = path.resolve(process.cwd(), 'specs.json');
|
|
6
|
+
|
|
7
|
+
// Check if the JSON file exists
|
|
8
|
+
if (!fs.existsSync(JSON_FILE)) {
|
|
9
|
+
console.error(`Error: ${JSON_FILE} does not exist.`);
|
|
10
|
+
process.exit(1);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// Ask the user for inputs
|
|
14
|
+
const readline = require('readline');
|
|
15
|
+
const rl = readline.createInterface({
|
|
16
|
+
input: process.stdin,
|
|
17
|
+
output: process.stdout,
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
let mode = ""; // To track if user wants to add or remove
|
|
21
|
+
|
|
22
|
+
// Function to prompt for mode
|
|
23
|
+
function askMode() {
|
|
24
|
+
rl.question('Would you like to add, remove, or view entries? (add/remove/view): ', (answer) => {
|
|
25
|
+
mode = answer.trim().toLowerCase();
|
|
26
|
+
if (mode === 'add' || mode === 'a') {
|
|
27
|
+
askQuestion(0);
|
|
28
|
+
} else if (mode === 'remove' || mode === 'r') {
|
|
29
|
+
askRemoveEntry();
|
|
30
|
+
} else if (mode === 'view' || mode === 'v') {
|
|
31
|
+
showReferences();
|
|
32
|
+
} else {
|
|
33
|
+
console.log('Invalid option. Please enter add, remove, or view.');
|
|
34
|
+
askMode();
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Questions for adding an entry
|
|
40
|
+
const questions = [
|
|
41
|
+
'Enter URL of the repository: ',
|
|
42
|
+
'Enter URL of the GitHub page: ',
|
|
43
|
+
'Enter the directory where the terms can be found: ',
|
|
44
|
+
'Enter short unique name: ',
|
|
45
|
+
];
|
|
46
|
+
|
|
47
|
+
let inputs = {};
|
|
48
|
+
|
|
49
|
+
// Function to ask questions for adding an entry
|
|
50
|
+
function askQuestion(index) {
|
|
51
|
+
if (index === questions.length) {
|
|
52
|
+
rl.close();
|
|
53
|
+
updateJSON();
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
rl.question(questions[index], (answer) => {
|
|
58
|
+
switch (index) {
|
|
59
|
+
case 0:
|
|
60
|
+
inputs.url = answer;
|
|
61
|
+
break;
|
|
62
|
+
case 1:
|
|
63
|
+
inputs.gh_page = answer;
|
|
64
|
+
break;
|
|
65
|
+
case 2:
|
|
66
|
+
inputs.terms_dir = answer;
|
|
67
|
+
break;
|
|
68
|
+
case 3:
|
|
69
|
+
inputs.external_spec = answer;
|
|
70
|
+
break;
|
|
71
|
+
}
|
|
72
|
+
askQuestion(index + 1);
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Function to prompt for removal
|
|
77
|
+
function askRemoveEntry() {
|
|
78
|
+
rl.question('Enter the short unique name of the entry you want to remove: ', (answer) => {
|
|
79
|
+
removeEntry(answer.trim());
|
|
80
|
+
rl.close();
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Function to show current external references
|
|
85
|
+
function showReferences() {
|
|
86
|
+
const data = JSON.parse(fs.readFileSync(JSON_FILE, 'utf8'));
|
|
87
|
+
console.log('\nCurrent external references (xref):\n\n');
|
|
88
|
+
|
|
89
|
+
data.specs[0].external_specs.forEach(spec => {
|
|
90
|
+
console.log('--- External Reference: ---');
|
|
91
|
+
console.log(`Short name: ${spec.external_spec}`);
|
|
92
|
+
console.log(`GitHub Page: ${spec.gh_page}`);
|
|
93
|
+
console.log(`URL: ${spec.url}`);
|
|
94
|
+
console.log(`Terms Directory: ${spec.terms_dir}`);
|
|
95
|
+
console.log('\n');
|
|
96
|
+
});
|
|
97
|
+
rl.close();
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Update the JSON file by adding an entry
|
|
101
|
+
function updateJSON() {
|
|
102
|
+
try {
|
|
103
|
+
const data = JSON.parse(fs.readFileSync(JSON_FILE, 'utf8'));
|
|
104
|
+
|
|
105
|
+
if (!data.specs || !Array.isArray(data.specs) || !data.specs[0].external_specs) {
|
|
106
|
+
console.error('Error: Invalid JSON structure. "specs[0].external_specs" is missing.');
|
|
107
|
+
process.exit(1);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const externalSpecs = data.specs[0].external_specs;
|
|
111
|
+
|
|
112
|
+
// Check if the external_spec already exists
|
|
113
|
+
const exists = externalSpecs.some(
|
|
114
|
+
(entry) => entry.external_spec === inputs.external_spec
|
|
115
|
+
);
|
|
116
|
+
|
|
117
|
+
if (exists) {
|
|
118
|
+
console.log(
|
|
119
|
+
`Entry with external_spec "${inputs.external_spec}" already exists. No changes made.`
|
|
120
|
+
);
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Add the new entry if it doesn't exist
|
|
125
|
+
externalSpecs.push(inputs);
|
|
126
|
+
|
|
127
|
+
fs.writeFileSync(JSON_FILE, JSON.stringify(data, null, 2), 'utf8');
|
|
128
|
+
// console.log(`Updated ${JSON_FILE} successfully.`);
|
|
129
|
+
console.log(`Updated successfully.`);
|
|
130
|
+
} catch (error) {
|
|
131
|
+
console.error(`Error: Failed to update ${JSON_FILE}.`, error.message);
|
|
132
|
+
process.exit(1);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Remove an entry based on `external_spec`
|
|
137
|
+
function removeEntry(externalSpec) {
|
|
138
|
+
try {
|
|
139
|
+
const data = JSON.parse(fs.readFileSync(JSON_FILE, 'utf8'));
|
|
140
|
+
|
|
141
|
+
if (!data.specs || !Array.isArray(data.specs) || !data.specs[0].external_specs) {
|
|
142
|
+
console.error('Error: Invalid JSON structure. "specs[0].external_specs" is missing.');
|
|
143
|
+
process.exit(1);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const externalSpecs = data.specs[0].external_specs;
|
|
147
|
+
|
|
148
|
+
// Filter out the entry with the matching external_spec
|
|
149
|
+
const filteredSpecs = externalSpecs.filter(
|
|
150
|
+
(entry) => entry.external_spec !== externalSpec
|
|
151
|
+
);
|
|
152
|
+
|
|
153
|
+
if (filteredSpecs.length === externalSpecs.length) {
|
|
154
|
+
console.log(`No entry found with external_spec "${externalSpec}".`);
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// Update the JSON structure
|
|
159
|
+
data.specs[0].external_specs = filteredSpecs;
|
|
160
|
+
|
|
161
|
+
fs.writeFileSync(JSON_FILE, JSON.stringify(data, null, 2), 'utf8');
|
|
162
|
+
// console.log(`Removed entry with external_spec "${externalSpec}" successfully.`);
|
|
163
|
+
console.log(`Removed entry successfully.`);
|
|
164
|
+
} catch (error) {
|
|
165
|
+
console.error(`Error: Failed to update ${JSON_FILE}.`, error.message);
|
|
166
|
+
process.exit(1);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// Start by asking the mode
|
|
171
|
+
askMode();
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file This script is responsible for fetching the latest commit hash of term files from the GitHub API and generating both a JavaScript file and a JSON file containing the data for the cross-references (xrefs).
|
|
3
|
+
*
|
|
4
|
+
* The generated JavaScript file is included in the HTML output of the specification, serving as a data source for the JavaScript code embedded in the HTML file.
|
|
5
|
+
*
|
|
6
|
+
* Additionally, the data is written to a JSON file for further processing or usage. This ensures that the xref data is available in both JavaScript and JSON formats, providing flexibility for different use cases.
|
|
7
|
+
*
|
|
8
|
+
* @author Kor Dwarshuis
|
|
9
|
+
* @version 1.0.0
|
|
10
|
+
* @since 2024-06-09
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
function collectExternalReferences(options = {}) {
|
|
15
|
+
require('dotenv').config();
|
|
16
|
+
const fs = require('fs-extra');
|
|
17
|
+
const readlineSync = require('readline-sync');
|
|
18
|
+
const config = fs.readJsonSync('specs.json');
|
|
19
|
+
const externalSpecsRepos = config.specs[0].external_specs;
|
|
20
|
+
const GITHUB_API_TOKEN = process.env.GITHUB_API_TOKEN;
|
|
21
|
+
|
|
22
|
+
const explanationPAT =
|
|
23
|
+
`\n SPEC-UP-T: No GitHub Personal Access Token (PAT) was found.
|
|
24
|
+
|
|
25
|
+
GitHub requires you to set up a PAT to retrieve external references.
|
|
26
|
+
|
|
27
|
+
There is no point in continuing without a PAT, so we stop here.
|
|
28
|
+
|
|
29
|
+
Find instructions on how to get a PAT at https://trustoverip.github.io/spec-up-t-website/docs/getting-started/github-token
|
|
30
|
+
|
|
31
|
+
`;
|
|
32
|
+
|
|
33
|
+
const explanationNoExternalReferences =
|
|
34
|
+
`\n SPEC-UP-T: No external references were found in the specs.json file.
|
|
35
|
+
|
|
36
|
+
There is no point in continuing without external references, so we stop here.
|
|
37
|
+
|
|
38
|
+
Please add external references to the specs.json file that you will find at the root of your project.
|
|
39
|
+
|
|
40
|
+
`;
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
// First do some checks
|
|
44
|
+
|
|
45
|
+
// Do not run the script if the GitHub API token is not set
|
|
46
|
+
if (!GITHUB_API_TOKEN) {
|
|
47
|
+
console.log(explanationPAT);
|
|
48
|
+
const userInput = readlineSync.question('Press any key');
|
|
49
|
+
|
|
50
|
+
// React to user pressing any key
|
|
51
|
+
if (userInput.trim() !== '') {
|
|
52
|
+
console.log('Stopping...');
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
else if (externalSpecsRepos.length === 0) {
|
|
58
|
+
// Check if the URLs for the external specs repositories are valid, and prompt the user to abort if they are not.
|
|
59
|
+
console.log(explanationNoExternalReferences);
|
|
60
|
+
const userInput = readlineSync.question('Press any key');
|
|
61
|
+
|
|
62
|
+
// React to user pressing any key
|
|
63
|
+
if (userInput.trim() !== '') {
|
|
64
|
+
console.log('Stopping...');
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
} else {
|
|
68
|
+
main();
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function main() {
|
|
72
|
+
const { processXTrefsData } = require('./collectExternalReferences/processXTrefsData.js');
|
|
73
|
+
const { doesUrlExist } = require('./utils/doesUrlExist.js');
|
|
74
|
+
|
|
75
|
+
// Check if the URLs for the external specs repositories are valid, and prompt the user to abort if they are not.
|
|
76
|
+
externalSpecsRepos.forEach(repo => {
|
|
77
|
+
doesUrlExist(repo.url, repo.terms_dir).then(exists => {
|
|
78
|
+
if (!exists) {
|
|
79
|
+
const userInput = readlineSync.question(
|
|
80
|
+
` SPEC-UP-T: This external reference is not a valid URL:
|
|
81
|
+
Repository: ${repo.url},
|
|
82
|
+
|
|
83
|
+
Terms directory: ${repo.terms_dir}
|
|
84
|
+
|
|
85
|
+
Please fix the external references in the specs.json file that you will find at the root of your project.
|
|
86
|
+
|
|
87
|
+
Do you want to stop? (yes/no): `);
|
|
88
|
+
if (userInput.toLowerCase() === 'yes' || userInput.toLowerCase() === 'y') {
|
|
89
|
+
console.log('Stopping...');
|
|
90
|
+
process.exit(1);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}).catch(error => {
|
|
94
|
+
console.error('\n SPEC-UP-T:Error checking URL existence:', error);
|
|
95
|
+
});
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
// Collect all directories that contain files with a term and definition
|
|
99
|
+
// This maps over the specs in the config file and constructs paths to directories
|
|
100
|
+
// where the term definition files are located.
|
|
101
|
+
const specTermsDirectories = config.specs.map(spec => spec.spec_directory + '/' + spec.spec_terms_directory);
|
|
102
|
+
|
|
103
|
+
// Ensure that the 'output' directory exists, creating it if necessary.
|
|
104
|
+
if (!fs.existsSync('output')) {
|
|
105
|
+
fs.mkdirSync('output');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Ensure that the 'output/xtrefs-history' directory exists, creating it if necessary.
|
|
109
|
+
if (!fs.existsSync('output/xtrefs-history')) {
|
|
110
|
+
fs.mkdirSync('output/xtrefs-history');
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Define paths for various output files, including JSON and JS files.
|
|
114
|
+
const outputPathJSON = 'output/xtrefs-data.json';
|
|
115
|
+
const outputPathJS = 'output/xtrefs-data.js';
|
|
116
|
+
const outputPathJSTimeStamped = 'output/xtrefs-history/xtrefs-data-' + Date.now() + '.js';
|
|
117
|
+
|
|
118
|
+
// Function to extend xtref objects with additional information, such as repository URL and directory information.
|
|
119
|
+
function extendXTrefs(config, xtrefs) {
|
|
120
|
+
if (config.specs[0].external_specs_repos) {
|
|
121
|
+
console.log("\n SPEC-UP-T: PLEASE NOTE: Your specs.json file is outdated (not your fault, we changed something). Use this one: https://github.com/trustoverip/spec-up-t-starter-pack/blob/main/spec-up-t-boilerplate/specs.json or e-mail kor@dwarshuis.com for help. \n");
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
xtrefs.forEach(xtref => {
|
|
126
|
+
config.specs.forEach(spec => {
|
|
127
|
+
// Loop through "external_specs" to find the repository URL for each xtref
|
|
128
|
+
xtref.repoUrl = null;
|
|
129
|
+
xtref.terms_dir = null;
|
|
130
|
+
xtref.owner = null;
|
|
131
|
+
xtref.repo = null;
|
|
132
|
+
|
|
133
|
+
spec.external_specs.forEach(repo => {
|
|
134
|
+
if (repo.external_spec === xtref.externalSpec) {
|
|
135
|
+
xtref.repoUrl = repo.url;
|
|
136
|
+
xtref.terms_dir = repo.terms_dir;
|
|
137
|
+
const urlParts = new URL(xtref.repoUrl).pathname.split('/');
|
|
138
|
+
xtref.owner = urlParts[1];
|
|
139
|
+
xtref.repo = urlParts[2];
|
|
140
|
+
xtref.avatarUrl = repo.avatar_url;
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
// Loop through "external_specs" to find the site URL for each xtref
|
|
145
|
+
|
|
146
|
+
xtref.site = null;
|
|
147
|
+
if (spec.external_specs) {
|
|
148
|
+
spec.external_specs.forEach(externalSpec => {
|
|
149
|
+
const key = Object.keys(externalSpec)[0];
|
|
150
|
+
if (key === xtref.externalSpec) {
|
|
151
|
+
xtref.site = externalSpec[key];
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Function to check if an xtref is in the markdown content
|
|
160
|
+
function isXTrefInMarkdown(xtref, markdownContent) {
|
|
161
|
+
// const regex = new RegExp(`\\[\\[xref:${xref.term}\\]\\]`, 'g');
|
|
162
|
+
const regex = new RegExp(`\\[\\[(?:x|t)ref:${xtref.term}\\]\\]`, 'g');
|
|
163
|
+
const result = regex.test(markdownContent);
|
|
164
|
+
return result;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Function to process and clean up xref / tref strings found in the markdown file, returning an object with `externalSpec` and `term` properties.
|
|
168
|
+
function processXTref(xtref) {
|
|
169
|
+
let [externalSpec, term] = xtref.replace(/\[\[(?:xref|tref):/, '').replace(/\]\]/, '').trim().split(/,/, 2);
|
|
170
|
+
return {
|
|
171
|
+
externalSpec: externalSpec.trim(),
|
|
172
|
+
term: term.trim()
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// Initialize an object to store all xtrefs.
|
|
177
|
+
let allXTrefs = { xtrefs: [] };
|
|
178
|
+
|
|
179
|
+
// If the output JSON file exists, load its data.
|
|
180
|
+
if (fs.existsSync(outputPathJSON)) {
|
|
181
|
+
const existingXTrefs = fs.readJsonSync(outputPathJSON);
|
|
182
|
+
if (existingXTrefs && existingXTrefs.xtrefs) {
|
|
183
|
+
allXTrefs = existingXTrefs;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// Collect all markdown content
|
|
188
|
+
let allMarkdownContent = '';
|
|
189
|
+
|
|
190
|
+
// Read all main repo Markdown files from a list of directories and concatenate their content into a single string.
|
|
191
|
+
specTermsDirectories.forEach(specDirectory => {
|
|
192
|
+
fs.readdirSync(specDirectory).forEach(file => {
|
|
193
|
+
if (file.endsWith('.md')) {
|
|
194
|
+
const markdown = fs.readFileSync(`${specDirectory}/${file}`, 'utf8');
|
|
195
|
+
allMarkdownContent += markdown;
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
// Remove existing entries if not in the combined markdown content
|
|
201
|
+
allXTrefs.xtrefs = allXTrefs.xtrefs.filter(existingXTref => {
|
|
202
|
+
return isXTrefInMarkdown(existingXTref, allMarkdownContent);
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
// Add new entries if they are in the markdown
|
|
206
|
+
const regex = /\[\[(?:xref|tref):.*?\]\]/g;
|
|
207
|
+
|
|
208
|
+
// `regex` is the regular expression object, and `allMarkdownContent` is the string being tested. The test method returns a boolean value: true if the pattern is found within the string, and false otherwise.
|
|
209
|
+
if (regex.test(allMarkdownContent)) {
|
|
210
|
+
const xtrefs = allMarkdownContent.match(regex);
|
|
211
|
+
xtrefs.forEach(xtref => {
|
|
212
|
+
const newXTrefObj = processXTref(xtref);
|
|
213
|
+
// Ensure that newXTrefObj is only added to the xtrefs array if there isn't already an object with the same term and externalSpec properties. This helps maintain the uniqueness of entries in the array based on these two properties.
|
|
214
|
+
if (!allXTrefs.xtrefs.some(existingXTref =>
|
|
215
|
+
existingXTref.term === newXTrefObj.term && existingXTref.externalSpec === newXTrefObj.externalSpec)) {
|
|
216
|
+
allXTrefs.xtrefs.push(newXTrefObj);
|
|
217
|
+
}
|
|
218
|
+
});
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
// Example at this point:
|
|
222
|
+
// allXTrefs.xtrefs: [
|
|
223
|
+
// { externalSpec: 'kmg-1', term: 'authentic-chained-data-container' },
|
|
224
|
+
// ]
|
|
225
|
+
|
|
226
|
+
// Extend each xref with additional data and fetch commit information from GitHub.
|
|
227
|
+
extendXTrefs(config, allXTrefs.xtrefs);
|
|
228
|
+
|
|
229
|
+
// Example at this point:
|
|
230
|
+
// allXTrefs.xtrefs: [
|
|
231
|
+
// {
|
|
232
|
+
// externalSpec: 'kmg-1',
|
|
233
|
+
// term: 'authentic-chained-data-container',
|
|
234
|
+
// repoUrl: 'https://github.com/henkvancann/keri-main-glossary',
|
|
235
|
+
// terms_dir: 'spec/terms-definitions',
|
|
236
|
+
// owner: 'henkvancann',
|
|
237
|
+
// repo: 'keri-main-glossary',
|
|
238
|
+
// site: null
|
|
239
|
+
// }
|
|
240
|
+
// ]
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
processXTrefsData(allXTrefs, GITHUB_API_TOKEN, outputPathJSON, outputPathJS, outputPathJSTimeStamped, options);
|
|
244
|
+
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
module.exports = {
|
|
251
|
+
collectExternalReferences
|
|
252
|
+
}
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
const crypto = require('crypto'); // For generating cache keys
|
|
4
|
+
const isLineWithDefinition = require('../utils/isLineWithDefinition').isLineWithDefinition;
|
|
5
|
+
const { addPath, getPath, getAllPaths } = require('../config/paths');
|
|
6
|
+
|
|
7
|
+
// Directory to store cached files
|
|
8
|
+
const CACHE_DIR = getPath('githubcache');
|
|
9
|
+
|
|
10
|
+
// Helper function to generate a cache key
|
|
11
|
+
function generateCacheKey(...args) {
|
|
12
|
+
const hash = crypto.createHash('md5').update(args.join('-')).digest('hex');
|
|
13
|
+
return hash;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
async function fetchTermsFromGitHubRepository(GITHUB_API_TOKEN, searchString, owner, repo, subdirectory) {
|
|
17
|
+
const { Octokit } = await import("octokit");
|
|
18
|
+
const { throttling } = await import("@octokit/plugin-throttling");
|
|
19
|
+
|
|
20
|
+
// Create a throttled Octokit instance
|
|
21
|
+
const ThrottledOctokit = Octokit.plugin(throttling);
|
|
22
|
+
const octokit = new ThrottledOctokit({
|
|
23
|
+
auth: GITHUB_API_TOKEN,
|
|
24
|
+
throttle: {
|
|
25
|
+
onRateLimit: (retryAfter, options) => {
|
|
26
|
+
console.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
|
|
27
|
+
if (options.request.retryCount <= 1) {
|
|
28
|
+
console.log(`Retrying after ${retryAfter} seconds...`);
|
|
29
|
+
return true;
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
onAbuseLimit: (retryAfter, options) => {
|
|
33
|
+
console.warn(`Abuse detected for request ${options.method} ${options.url}`);
|
|
34
|
+
},
|
|
35
|
+
onSecondaryRateLimit: (retryAfter, options) => {
|
|
36
|
+
console.warn(`Secondary rate limit hit for request ${options.method} ${options.url}`);
|
|
37
|
+
if (options.request.retryCount <= 1) {
|
|
38
|
+
console.log(`Retrying after ${retryAfter} seconds...`);
|
|
39
|
+
return true;
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
try {
|
|
46
|
+
// Generate a cache key for the search query
|
|
47
|
+
const searchCacheKey = generateCacheKey('search', searchString, owner, repo, subdirectory);
|
|
48
|
+
const searchCacheFilePath = path.join(CACHE_DIR, `${searchCacheKey}.json`);
|
|
49
|
+
|
|
50
|
+
let searchResponse;
|
|
51
|
+
|
|
52
|
+
// Check if the search response is already cached
|
|
53
|
+
if (fs.existsSync(searchCacheFilePath)) {
|
|
54
|
+
console.log(`Serving search results from cache: ${searchCacheFilePath}`);
|
|
55
|
+
searchResponse = JSON.parse(fs.readFileSync(searchCacheFilePath, 'utf-8'));
|
|
56
|
+
} else {
|
|
57
|
+
// Perform the search using Octokit with exact match
|
|
58
|
+
console.log(`Performing search and caching results: ${searchCacheFilePath}`);
|
|
59
|
+
searchResponse = await octokit.rest.search.code({
|
|
60
|
+
q: `"${searchString}" repo:${owner}/${repo} path:${subdirectory}`, // Exact match in subdirectory
|
|
61
|
+
headers: {
|
|
62
|
+
Accept: "application/vnd.github.v3.text-match+json", // Include text-match media type
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
// Cache the search response
|
|
67
|
+
fs.writeFileSync(searchCacheFilePath, JSON.stringify(searchResponse), 'utf-8');
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Log the search results
|
|
71
|
+
console.log(`Total matches for ${searchString} :`, searchResponse.data.total_count);
|
|
72
|
+
|
|
73
|
+
/*
|
|
74
|
+
|
|
75
|
+
Each item is a file that contains the search string one or more times. So if a search string is found in 'attribute-based-access-control.md' and 'abac.md', both files will be returned as separate items. Each item contains “text_matches”.
|
|
76
|
+
|
|
77
|
+
- text_matches can contain multiple objects if there are multiple matches in the file.
|
|
78
|
+
- fragment is a snippet of the file content around the matched search string, not the entire file content.
|
|
79
|
+
|
|
80
|
+
In example below:
|
|
81
|
+
|
|
82
|
+
- The total_count is 2, indicating there are two files that contain the search string.
|
|
83
|
+
- Each item in the items array represents a file.
|
|
84
|
+
- The text_matches array within each item contains objects representing different matches of the search string within the file.
|
|
85
|
+
- Each fragment is a snippet of the file content around the matched search string, not the entire file content.
|
|
86
|
+
|
|
87
|
+
{
|
|
88
|
+
"total_count": 2,
|
|
89
|
+
"items": [
|
|
90
|
+
{
|
|
91
|
+
"name": "example-file1.md",
|
|
92
|
+
"path": "docs/example-file1.md",
|
|
93
|
+
"sha": "abc123",
|
|
94
|
+
"url": "https://api.github.com/repositories/123456789/contents/docs/example-file1.md",
|
|
95
|
+
"git_url": "https://api.github.com/repositories/123456789/git/blobs/abc123",
|
|
96
|
+
"html_url": "https://github.com/owner/repo/blob/main/docs/example-file1.md",
|
|
97
|
+
"repository": {
|
|
98
|
+
"id": 123456789,
|
|
99
|
+
"name": "repo",
|
|
100
|
+
"full_name": "owner/repo",
|
|
101
|
+
"owner": {
|
|
102
|
+
"login": "owner",
|
|
103
|
+
"id": 12345,
|
|
104
|
+
"avatar_url": "https://avatars.githubusercontent.com/u/12345?v=4",
|
|
105
|
+
"url": "https://api.github.com/users/owner"
|
|
106
|
+
}
|
|
107
|
+
},
|
|
108
|
+
"text_matches": [
|
|
109
|
+
{
|
|
110
|
+
"object_url": "https://api.github.com/repositories/123456789/contents/docs/example-file1.md",
|
|
111
|
+
"object_type": "FileContent",
|
|
112
|
+
"property": "content",
|
|
113
|
+
"fragment": "This is an example content with the search string.",
|
|
114
|
+
"matches": [
|
|
115
|
+
{
|
|
116
|
+
"text": "search string",
|
|
117
|
+
"indices": [31, 44]
|
|
118
|
+
}
|
|
119
|
+
]
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
"object_url": "https://api.github.com/repositories/123456789/contents/docs/example-file1.md",
|
|
123
|
+
"object_type": "FileContent",
|
|
124
|
+
"property": "content",
|
|
125
|
+
"fragment": "Another occurrence of the search string in the file.",
|
|
126
|
+
"matches": [
|
|
127
|
+
{
|
|
128
|
+
"text": "search string",
|
|
129
|
+
"indices": [25, 38]
|
|
130
|
+
}
|
|
131
|
+
]
|
|
132
|
+
}
|
|
133
|
+
]
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
"name": "example-file2.md",
|
|
137
|
+
"path": "docs/example-file2.md",
|
|
138
|
+
"sha": "def456",
|
|
139
|
+
"url": "https://api.github.com/repositories/123456789/contents/docs/example-file2.md",
|
|
140
|
+
"git_url": "https://api.github.com/repositories/123456789/git/blobs/def456",
|
|
141
|
+
"html_url": "https://github.com/owner/repo/blob/main/docs/example-file2.md",
|
|
142
|
+
"repository": {
|
|
143
|
+
"id": 123456789,
|
|
144
|
+
"name": "repo",
|
|
145
|
+
"full_name": "owner/repo",
|
|
146
|
+
"owner": {
|
|
147
|
+
"login": "owner",
|
|
148
|
+
"id": 12345,
|
|
149
|
+
"avatar_url": "https://avatars.githubusercontent.com/u/12345?v=4",
|
|
150
|
+
"url": "https://api.github.com/users/owner"
|
|
151
|
+
}
|
|
152
|
+
},
|
|
153
|
+
"text_matches": [
|
|
154
|
+
{
|
|
155
|
+
"object_url": "https://api.github.com/repositories/123456789/contents/docs/example-file2.md",
|
|
156
|
+
"object_type": "FileContent",
|
|
157
|
+
"property": "content",
|
|
158
|
+
"fragment": "This file also contains the search string.",
|
|
159
|
+
"matches": [
|
|
160
|
+
{
|
|
161
|
+
"text": "search string",
|
|
162
|
+
"indices": [25, 38]
|
|
163
|
+
}
|
|
164
|
+
]
|
|
165
|
+
}
|
|
166
|
+
]
|
|
167
|
+
}
|
|
168
|
+
]
|
|
169
|
+
}
|
|
170
|
+
*/
|
|
171
|
+
|
|
172
|
+
for (const item of searchResponse.data.items) {
|
|
173
|
+
// Check if text_matches exists and is not empty
|
|
174
|
+
if (!item.text_matches || item.text_matches.length === 0) {
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Loop through each text match. Can contain multiple fragments
|
|
179
|
+
for (const match of item.text_matches) {
|
|
180
|
+
// Split the fragment into lines, lines can be empty ('')
|
|
181
|
+
const lines = match.fragment.split("\n");
|
|
182
|
+
for (const line of lines) {
|
|
183
|
+
if (isLineWithDefinition(line)) {
|
|
184
|
+
// Generate a unique cache key for the file
|
|
185
|
+
const fileCacheKey = generateCacheKey('file', owner, repo, item.path);
|
|
186
|
+
const fileCacheFilePath = path.join(CACHE_DIR, `${fileCacheKey}.txt`);
|
|
187
|
+
|
|
188
|
+
let fileContent;
|
|
189
|
+
|
|
190
|
+
// Check if the file is already cached
|
|
191
|
+
if (fs.existsSync(fileCacheFilePath)) {
|
|
192
|
+
console.log(`Serving file from cache: ${fileCacheFilePath}`);
|
|
193
|
+
fileContent = fs.readFileSync(fileCacheFilePath, 'utf-8');
|
|
194
|
+
} else {
|
|
195
|
+
// Fetch file content from GitHub
|
|
196
|
+
console.log(`Downloading and caching file: ${fileCacheFilePath}`);
|
|
197
|
+
try {
|
|
198
|
+
const fileContentResponse = await octokit.rest.repos.getContent({
|
|
199
|
+
owner: item.repository.owner.login, // Repository owner
|
|
200
|
+
repo: item.repository.name, // Repository name
|
|
201
|
+
path: item.path, // File path
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
// Decode the file content (it's base64-encoded)
|
|
205
|
+
if (fileContentResponse.data.content) {
|
|
206
|
+
fileContent = Buffer.from(fileContentResponse.data.content, "base64").toString("utf-8");
|
|
207
|
+
// Save the file to the cache
|
|
208
|
+
fs.writeFileSync(fileCacheFilePath, fileContent, 'utf-8');
|
|
209
|
+
} else {
|
|
210
|
+
// If the file is larger than 1 MB, GitHub's API will return a download URL instead of the content.
|
|
211
|
+
console.log("File is too large. Download URL:", fileContentResponse.data.download_url);
|
|
212
|
+
fileContent = "";
|
|
213
|
+
}
|
|
214
|
+
} catch (error) {
|
|
215
|
+
console.error(`Error fetching content for ${item.path}:`, error);
|
|
216
|
+
fileContent = ""; // Set content to an empty string if there's an error
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Attach the content to the item
|
|
221
|
+
item.content = fileContent;
|
|
222
|
+
|
|
223
|
+
// Return the item as soon as we find the correct line
|
|
224
|
+
return item;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
} catch (error) {
|
|
230
|
+
console.error("Error searching GitHub or fetching file content:", error);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// If no item is found, return null or undefined
|
|
234
|
+
return null;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
exports.fetchTermsFromGitHubRepository = fetchTermsFromGitHubRepository;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
const isLineWithDefinition = require('../utils/isLineWithDefinition').isLineWithDefinition;
|
|
2
|
+
|
|
1
3
|
function matchTerm(text, term) {
|
|
2
4
|
if (!text || typeof text !== 'string') {
|
|
3
5
|
// console.error('Invalid text:', text);
|
|
@@ -7,11 +9,10 @@ function matchTerm(text, term) {
|
|
|
7
9
|
|
|
8
10
|
const firstLine = text.split('\n')[0].trim();
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
if (!firstLine.startsWith('[[def:') || !firstLine.endsWith(']]')) {
|
|
12
|
+
if (isLineWithDefinition(firstLine) === false) {
|
|
12
13
|
console.log('String does not start with `[[def:` or end with `]]`');
|
|
13
14
|
return false;
|
|
14
|
-
}
|
|
15
|
+
};
|
|
15
16
|
|
|
16
17
|
// Remove `[[def:` from the beginning and `]]` from the end
|
|
17
18
|
let relevantPart = firstLine.slice(7, -2);
|