volunteer-contributions-fetch 2.3.3 → 2.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -22
- package/lib/github-fetcher.js +11 -1
- package/lib/github.js +50 -15
- package/lib/github.test.js +76 -8
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -119,28 +119,28 @@ This section documents all the possible configuration values. For examples see t
|
|
|
119
119
|
|
|
120
120
|
#### GitHub
|
|
121
121
|
|
|
122
|
-
| Field | Data Type | Required | Default | Description
|
|
123
|
-
| -------------------------- | --------- | -------- | ----------------------------------- |
|
|
124
|
-
| `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `github` object entirely.
|
|
125
|
-
| `allowPrivate` | boolean | No | false |
|
|
126
|
-
| `commentsEnabled` | boolean | No | true | Indicates whether fetching comments is enabled.
|
|
127
|
-
| `commitsEnabled` | boolean | No | true | Indicates whether fetching commits is enabled.
|
|
128
|
-
| `issuesEnabled` | boolean | No | true | Indicates whether fetching issues is enabled.
|
|
129
|
-
| `reviewsEnabled` | boolean | No | true | Indicates whether fetching reviews is enabled.
|
|
130
|
-
| `username` | string | Yes | - | Username on GitHub
|
|
131
|
-
| `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track
|
|
132
|
-
| `filter` | string | Yes | - | Regex to apply to each entry to filter for organization/repo names. For example: `^mozilla` to only include orgs and repos that start with "mozilla".
|
|
133
|
-
| `types` | Object | No | - | Object containing the different labels below
|
|
134
|
-
| `types.commit` | string | No | GitHub Commit | Label for entries related to GitHub commits
|
|
135
|
-
| `types.privateCommit` | string | No | Commit in private repository | Label for entries related to private GitHub commits
|
|
136
|
-
| `types.createdPR` | string | No | Created PR | Label for entries related to opening GitHub PRs
|
|
137
|
-
| `types.createdIssue` | string | No | Created Issue Report | Label for entries related to creating GitHub issues
|
|
138
|
-
| `types.commented` | string | No | Commented on an Issue | Label for entries related to commenting on an issue
|
|
139
|
-
| `types.commentedPR` | string | No | Commented on a Pull Request | Label for entries related to commenting on a GitHub PR (review)
|
|
140
|
-
| `types.approvedPR` | string | No | Approved a Pull Request | Label for entries related to approving a GitHub PR
|
|
141
|
-
| `types.changesRequestedPR` | string | No | Requested changes on a Pull Request | Label for entries related to requesting changes on a GitHub PR
|
|
142
|
-
| `types.reviewedPR` | string | No | Reviewed a Pull Request | Label for entries related to reviewing a GitHub PR (fallback)
|
|
143
|
-
| `delayMsPerRequest` | number | No | 2000 | Delay between each request to GitHub. You might hit rate limiting if you set this lower than default.
|
|
122
|
+
| Field | Data Type | Required | Default | Description |
|
|
123
|
+
| -------------------------- | --------- | -------- | ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
124
|
+
| `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `github` object entirely. |
|
|
125
|
+
| `allowPrivate` | boolean | No | false | Will throw if you pass a `GITHUB_TOKEN` with possible private repo scope. In most cases you will not want to have a token with private scope. Please think twice before enabling this config. When using fine-grained tokens, you need to set this to `true` manually as we can't detect the scopes. In that case, we can't guarantee anything! |
|
|
126
|
+
| `commentsEnabled` | boolean | No | true | Indicates whether fetching comments is enabled. |
|
|
127
|
+
| `commitsEnabled` | boolean | No | true | Indicates whether fetching commits is enabled. |
|
|
128
|
+
| `issuesEnabled` | boolean | No | true | Indicates whether fetching issues is enabled. |
|
|
129
|
+
| `reviewsEnabled` | boolean | No | true | Indicates whether fetching reviews is enabled. |
|
|
130
|
+
| `username` | string | Yes | - | Username on GitHub |
|
|
131
|
+
| `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track |
|
|
132
|
+
| `filter` | string | Yes | - | Regex to apply to each entry to filter for organization/repo names. For example: `^mozilla` to only include orgs and repos that start with "mozilla". |
|
|
133
|
+
| `types` | Object | No | - | Object containing the different labels below |
|
|
134
|
+
| `types.commit` | string | No | GitHub Commit | Label for entries related to GitHub commits |
|
|
135
|
+
| `types.privateCommit` | string | No | Commit in private repository | Label for entries related to private GitHub commits |
|
|
136
|
+
| `types.createdPR` | string | No | Created PR | Label for entries related to opening GitHub PRs |
|
|
137
|
+
| `types.createdIssue` | string | No | Created Issue Report | Label for entries related to creating GitHub issues |
|
|
138
|
+
| `types.commented` | string | No | Commented on an Issue | Label for entries related to commenting on an issue |
|
|
139
|
+
| `types.commentedPR` | string | No | Commented on a Pull Request | Label for entries related to commenting on a GitHub PR (review) |
|
|
140
|
+
| `types.approvedPR` | string | No | Approved a Pull Request | Label for entries related to approving a GitHub PR |
|
|
141
|
+
| `types.changesRequestedPR` | string | No | Requested changes on a Pull Request | Label for entries related to requesting changes on a GitHub PR |
|
|
142
|
+
| `types.reviewedPR` | string | No | Reviewed a Pull Request | Label for entries related to reviewing a GitHub PR (fallback) |
|
|
143
|
+
| `delayMsPerRequest` | number | No | 2000 | Delay between each request to GitHub. You might hit rate limiting if you set this lower than default. |
|
|
144
144
|
|
|
145
145
|
#### MediaWiki
|
|
146
146
|
|
package/lib/github-fetcher.js
CHANGED
|
@@ -2,9 +2,11 @@ const debug = require('debug')('contributions:github-fetcher');
|
|
|
2
2
|
const { Octokit } = require('@octokit/rest');
|
|
3
3
|
|
|
4
4
|
const RESULTS_PER_PAGE = 100;
|
|
5
|
+
const PRIVATE_SCOPE_ERROR = 'PRIVATE_SCOPE';
|
|
5
6
|
|
|
6
7
|
module.exports = {
|
|
7
8
|
RESULTS_PER_PAGE,
|
|
9
|
+
PRIVATE_SCOPE_ERROR,
|
|
8
10
|
searchCommits,
|
|
9
11
|
searchIssues,
|
|
10
12
|
listReviews,
|
|
@@ -76,8 +78,16 @@ async function listComments(options) {
|
|
|
76
78
|
async function throwIfTokenHasPrivateRepoScope() {
|
|
77
79
|
const octokit = getOctokit();
|
|
78
80
|
const response = await octokit.rest.meta.root();
|
|
81
|
+
// If we don't have the scopes, such as for fine-grained
|
|
82
|
+
// access tokens, we error as well. We can't guarantee
|
|
83
|
+
// that the passed token is not indeed allowing private
|
|
84
|
+
// access.
|
|
85
|
+
if (!response.headers['x-oauth-scopes']) {
|
|
86
|
+
throw new Error(PRIVATE_SCOPE_ERROR);
|
|
87
|
+
}
|
|
88
|
+
|
|
79
89
|
const scopes = response.headers['x-oauth-scopes'].split(', ');
|
|
80
90
|
if (scopes.includes('repo')) {
|
|
81
|
-
throw new Error(
|
|
91
|
+
throw new Error(PRIVATE_SCOPE_ERROR);
|
|
82
92
|
}
|
|
83
93
|
}
|
package/lib/github.js
CHANGED
|
@@ -54,12 +54,12 @@ async function gather({ github }, existingContributions = []) {
|
|
|
54
54
|
const latestCommit = existingContributions.find((contribution) => {
|
|
55
55
|
return contribution.source === GITHUB_COMMIT_CATEGORY;
|
|
56
56
|
}) || { createdAt: github.stopDate };
|
|
57
|
-
const latestIssue = existingContributions.find(
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
)
|
|
62
|
-
|
|
57
|
+
const latestIssue = existingContributions.find(
|
|
58
|
+
(contribution) => contribution.source === GITHUB_ISSUES_CREATED_CATEGORY
|
|
59
|
+
) || { createdAt: github.stopDate };
|
|
60
|
+
const latestPR = existingContributions.find(
|
|
61
|
+
(contribution) => contribution.source === GITHUB_PR_CREATED_CATEGORY
|
|
62
|
+
) || { createdAt: github.stopDate };
|
|
63
63
|
const latestReview = existingContributions.find((contribution) => {
|
|
64
64
|
return contribution.source === GITHUB_REVIEW_CATEGORY;
|
|
65
65
|
}) || { createdAt: github.stopDate };
|
|
@@ -70,10 +70,10 @@ async function gather({ github }, existingContributions = []) {
|
|
|
70
70
|
try {
|
|
71
71
|
await fetcher.throwIfTokenHasPrivateRepoScope();
|
|
72
72
|
} catch (error) {
|
|
73
|
-
if (error.message ===
|
|
73
|
+
if (error.message === fetcher.PRIVATE_SCOPE_ERROR) {
|
|
74
74
|
if (!github.allowPrivate) {
|
|
75
75
|
throw new Error(
|
|
76
|
-
'GITHUB_TOKEN
|
|
76
|
+
'GITHUB_TOKEN might have private repo scope, but the config does not specify `allowPrivate` to fetch private repos. Are you sure you want to fetch private information? This might also be because you passed a fine-grained token. In that case we can not check the permissions it has and we assume it is private. You can double check the permissions and if you are sure it is correct, you may set `allowPrivate` to true.'
|
|
77
77
|
);
|
|
78
78
|
}
|
|
79
79
|
} else {
|
|
@@ -83,9 +83,16 @@ async function gather({ github }, existingContributions = []) {
|
|
|
83
83
|
|
|
84
84
|
const commits = await processCommits(github, latestCommit.createdAt);
|
|
85
85
|
const issues = await processIssues(github, latestIssue.createdAt);
|
|
86
|
+
const prs = await processPRs(github, latestPR.createdAt);
|
|
86
87
|
const reviews = await processReviews(github, latestReview.createdAt);
|
|
87
88
|
const comments = await processComments(github, latestComment.createdAt);
|
|
88
|
-
const contributions = [
|
|
89
|
+
const contributions = [
|
|
90
|
+
...commits,
|
|
91
|
+
...issues,
|
|
92
|
+
...prs,
|
|
93
|
+
...reviews,
|
|
94
|
+
...comments,
|
|
95
|
+
];
|
|
89
96
|
|
|
90
97
|
debug('Finished gathering contributions');
|
|
91
98
|
return contributions;
|
|
@@ -120,6 +127,35 @@ async function processIssues(github, latestCreationDate) {
|
|
|
120
127
|
});
|
|
121
128
|
}
|
|
122
129
|
|
|
130
|
+
async function processPRs(github, latestCreationDate) {
|
|
131
|
+
if (
|
|
132
|
+
typeof github.issuesEnabled !== 'undefined' &&
|
|
133
|
+
github.issuesEnabled === false
|
|
134
|
+
) {
|
|
135
|
+
debug('Fetching PRs is disabled through `issuesEnabled`, skipping!');
|
|
136
|
+
return [];
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
debug('Getting PRs from Github');
|
|
140
|
+
const allPRs = [];
|
|
141
|
+
|
|
142
|
+
const lowerBoundDate = new Date(latestCreationDate);
|
|
143
|
+
const generatedPeriod = periodGenerator(github.stopDate, lowerBoundDate);
|
|
144
|
+
for await (const periodChunk of generatedPeriod) {
|
|
145
|
+
const currentPage = pageGenerator(github, periodChunk, (period, page) =>
|
|
146
|
+
getPRPage(github, 'author', period, page)
|
|
147
|
+
);
|
|
148
|
+
for await (const prs of currentPage) {
|
|
149
|
+
debug(`Got ${prs.length} PRs`);
|
|
150
|
+
allPRs.push(prs);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
return allPRs.flat().filter((issue) => {
|
|
155
|
+
return new Date(issue.createdAt) > new Date(latestCreationDate);
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
|
|
123
159
|
async function processCommits(github, latestCreationDate) {
|
|
124
160
|
if (
|
|
125
161
|
typeof github.commitsEnabled !== 'undefined' &&
|
|
@@ -173,7 +209,7 @@ async function processReviews(github, latestCreationDate) {
|
|
|
173
209
|
const currentPage = pageGenerator(
|
|
174
210
|
github,
|
|
175
211
|
periodChunk,
|
|
176
|
-
(period, page) => getPRPage(github, period, page),
|
|
212
|
+
(period, page) => getPRPage(github, 'reviewed-by', period, page),
|
|
177
213
|
{ skipProcessing: true }
|
|
178
214
|
);
|
|
179
215
|
for await (const prs of currentPage) {
|
|
@@ -365,9 +401,8 @@ async function getIssuePage(github, relationship, issuesPeriod, page) {
|
|
|
365
401
|
debug(`Getting issues page ${page} between ${after} and ${before}`);
|
|
366
402
|
await waitFor(github.delayMsPerRequest ?? DELAY_FETCHING_MS);
|
|
367
403
|
try {
|
|
368
|
-
const additionalFields = relationship === 'commenter' ? '+is:issue' : '';
|
|
369
404
|
const items = await fetcher.searchIssues({
|
|
370
|
-
q: `${relationship}:${github.username}+created:${after}..${before}
|
|
405
|
+
q: `${relationship}:${github.username}+created:${after}..${before}+is:issue`,
|
|
371
406
|
sort: 'created',
|
|
372
407
|
page,
|
|
373
408
|
});
|
|
@@ -383,14 +418,14 @@ async function getIssuePage(github, relationship, issuesPeriod, page) {
|
|
|
383
418
|
}
|
|
384
419
|
}
|
|
385
420
|
|
|
386
|
-
async function getPRPage(github, reviewPeriod, page) {
|
|
421
|
+
async function getPRPage(github, relationship, reviewPeriod, page) {
|
|
387
422
|
const [before, after] = reviewPeriod;
|
|
388
423
|
|
|
389
|
-
debug(`Getting
|
|
424
|
+
debug(`Getting PR page ${page} between ${after} and ${before}`);
|
|
390
425
|
await waitFor(github.delayMsPerRequest ?? DELAY_FETCHING_MS);
|
|
391
426
|
try {
|
|
392
427
|
const items = await fetcher.searchIssues({
|
|
393
|
-
q:
|
|
428
|
+
q: `${relationship}:${github.username}+created:${after}..${before}+is:pull-request`,
|
|
394
429
|
sort: 'created',
|
|
395
430
|
page,
|
|
396
431
|
});
|
package/lib/github.test.js
CHANGED
|
@@ -407,22 +407,22 @@ test.serial('should fetch issues until stop date', async (t) => {
|
|
|
407
407
|
|
|
408
408
|
await github.gather(commitConfig);
|
|
409
409
|
t.deepEqual(fetcher.searchIssues.getCall(0).args[0], {
|
|
410
|
-
q: `author:${config.github.username}+created:2022-03-01..2022-03-31`,
|
|
410
|
+
q: `author:${config.github.username}+created:2022-03-01..2022-03-31+is:issue`,
|
|
411
411
|
sort: 'created',
|
|
412
412
|
page: 1,
|
|
413
413
|
});
|
|
414
414
|
t.deepEqual(fetcher.searchIssues.getCall(1).args[0], {
|
|
415
|
-
q: `author:${config.github.username}+created:2022-03-01..2022-03-31`,
|
|
415
|
+
q: `author:${config.github.username}+created:2022-03-01..2022-03-31+is:issue`,
|
|
416
416
|
sort: 'created',
|
|
417
417
|
page: 2,
|
|
418
418
|
});
|
|
419
419
|
t.deepEqual(fetcher.searchIssues.getCall(2).args[0], {
|
|
420
|
-
q: `author:${config.github.username}+created:2022-02-01..2022-02-28`,
|
|
420
|
+
q: `author:${config.github.username}+created:2022-02-01..2022-02-28+is:issue`,
|
|
421
421
|
sort: 'created',
|
|
422
422
|
page: 1,
|
|
423
423
|
});
|
|
424
424
|
t.deepEqual(fetcher.searchIssues.getCall(3).args[0], {
|
|
425
|
-
q: `author:${config.github.username}+created:2022-01-01..2022-01-31`,
|
|
425
|
+
q: `author:${config.github.username}+created:2022-01-01..2022-01-31+is:issue`,
|
|
426
426
|
sort: 'created',
|
|
427
427
|
page: 1,
|
|
428
428
|
});
|
|
@@ -611,6 +611,74 @@ test.serial('should ignore issues not matching filter', async (t) => {
|
|
|
611
611
|
t.is(result.length, 0);
|
|
612
612
|
});
|
|
613
613
|
|
|
614
|
+
test.serial('should fetch PRs until stop date', async (t) => {
|
|
615
|
+
const commitConfig = {
|
|
616
|
+
github: {
|
|
617
|
+
...config.github,
|
|
618
|
+
commitsEnabled: false,
|
|
619
|
+
reviewsEnabled: false,
|
|
620
|
+
},
|
|
621
|
+
};
|
|
622
|
+
|
|
623
|
+
fetcher.searchIssues.resolves([]);
|
|
624
|
+
// We want to fetch 2 pages for the first period, and one page for the subsequent periods.
|
|
625
|
+
// We expect 3 periods to be queried based on the passed stopDate and the faked clock.
|
|
626
|
+
// Therefore PRs start with 3 as before it searches for issues
|
|
627
|
+
fetcher.searchIssues
|
|
628
|
+
.onCall(3)
|
|
629
|
+
.resolves(
|
|
630
|
+
new Array(100).fill(0).map(() => ({ createdAt: new Date('2022-03-01') }))
|
|
631
|
+
);
|
|
632
|
+
fetcher.searchIssues
|
|
633
|
+
.onCall(4)
|
|
634
|
+
.resolves(
|
|
635
|
+
new Array(5).fill(0).map(() => ({ createdAt: new Date('2022-02-01') }))
|
|
636
|
+
);
|
|
637
|
+
fetcher.searchIssues
|
|
638
|
+
.onCall(5)
|
|
639
|
+
.resolves(
|
|
640
|
+
new Array(5).fill(0).map(() => ({ createdAt: new Date('2022-01-01') }))
|
|
641
|
+
);
|
|
642
|
+
|
|
643
|
+
await github.gather(commitConfig);
|
|
644
|
+
t.deepEqual(fetcher.searchIssues.getCall(3).args[0], {
|
|
645
|
+
q: `author:${config.github.username}+created:2022-03-01..2022-03-31+is:pull-request`,
|
|
646
|
+
sort: 'created',
|
|
647
|
+
page: 1,
|
|
648
|
+
});
|
|
649
|
+
t.deepEqual(fetcher.searchIssues.getCall(4).args[0], {
|
|
650
|
+
q: `author:${config.github.username}+created:2022-03-01..2022-03-31+is:pull-request`,
|
|
651
|
+
sort: 'created',
|
|
652
|
+
page: 2,
|
|
653
|
+
});
|
|
654
|
+
t.deepEqual(fetcher.searchIssues.getCall(5).args[0], {
|
|
655
|
+
q: `author:${config.github.username}+created:2022-02-01..2022-02-28+is:pull-request`,
|
|
656
|
+
sort: 'created',
|
|
657
|
+
page: 1,
|
|
658
|
+
});
|
|
659
|
+
t.deepEqual(fetcher.searchIssues.getCall(6).args[0], {
|
|
660
|
+
q: `author:${config.github.username}+created:2022-01-01..2022-01-31+is:pull-request`,
|
|
661
|
+
sort: 'created',
|
|
662
|
+
page: 1,
|
|
663
|
+
});
|
|
664
|
+
});
|
|
665
|
+
|
|
666
|
+
test.serial('should not fetch PRs if disabled', async (t) => {
|
|
667
|
+
const commitConfig = {
|
|
668
|
+
github: {
|
|
669
|
+
...config.github,
|
|
670
|
+
commitsEnabled: false,
|
|
671
|
+
issuesEnabled: false,
|
|
672
|
+
reviewsEnabled: false,
|
|
673
|
+
filter: 'contribution-test',
|
|
674
|
+
stopDate: new Date('2022-03-01'),
|
|
675
|
+
},
|
|
676
|
+
};
|
|
677
|
+
|
|
678
|
+
await github.gather(commitConfig);
|
|
679
|
+
t.is(fetcher.searchCommits.callCount, 0);
|
|
680
|
+
});
|
|
681
|
+
|
|
614
682
|
test.serial('should fetch reviews until stop date', async (t) => {
|
|
615
683
|
const commitConfig = {
|
|
616
684
|
github: {
|
|
@@ -666,22 +734,22 @@ test.serial('should fetch reviews until stop date', async (t) => {
|
|
|
666
734
|
|
|
667
735
|
await github.gather(commitConfig);
|
|
668
736
|
t.deepEqual(fetcher.searchIssues.getCall(0).args[0], {
|
|
669
|
-
q: `reviewed-by:${config.github.username}+created:2022-03-01..2022-03-31`,
|
|
737
|
+
q: `reviewed-by:${config.github.username}+created:2022-03-01..2022-03-31+is:pull-request`,
|
|
670
738
|
sort: 'created',
|
|
671
739
|
page: 1,
|
|
672
740
|
});
|
|
673
741
|
t.deepEqual(fetcher.searchIssues.getCall(1).args[0], {
|
|
674
|
-
q: `reviewed-by:${config.github.username}+created:2022-03-01..2022-03-31`,
|
|
742
|
+
q: `reviewed-by:${config.github.username}+created:2022-03-01..2022-03-31+is:pull-request`,
|
|
675
743
|
sort: 'created',
|
|
676
744
|
page: 2,
|
|
677
745
|
});
|
|
678
746
|
t.deepEqual(fetcher.searchIssues.getCall(2).args[0], {
|
|
679
|
-
q: `reviewed-by:${config.github.username}+created:2022-02-01..2022-02-28`,
|
|
747
|
+
q: `reviewed-by:${config.github.username}+created:2022-02-01..2022-02-28+is:pull-request`,
|
|
680
748
|
sort: 'created',
|
|
681
749
|
page: 1,
|
|
682
750
|
});
|
|
683
751
|
t.deepEqual(fetcher.searchIssues.getCall(3).args[0], {
|
|
684
|
-
q: `reviewed-by:${config.github.username}+created:2022-01-01..2022-01-31`,
|
|
752
|
+
q: `reviewed-by:${config.github.username}+created:2022-01-01..2022-01-31+is:pull-request`,
|
|
685
753
|
sort: 'created',
|
|
686
754
|
page: 1,
|
|
687
755
|
});
|