volunteer-contributions-fetch 2.3.3 → 2.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -119,28 +119,28 @@ This section documents all the possible configuration values. For examples see t
119
119
 
120
120
  #### GitHub
121
121
 
122
- | Field | Data Type | Required | Default | Description |
123
- | -------------------------- | --------- | -------- | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
124
- | `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `github` object entirely. |
125
- | `allowPrivate` | boolean | No | false | Indicates whether fetching from private repositories is allowed. If not set, the script will throw if you pass a `GITHUB_TOKEN` with private repo scope. In most cases you will not want to have a token with private scope as you do not want to leak any private information in your contribution overview. Please think twice before enabling this config. |
126
- | `commentsEnabled` | boolean | No | true | Indicates whether fetching comments is enabled. |
127
- | `commitsEnabled` | boolean | No | true | Indicates whether fetching commits is enabled. |
128
- | `issuesEnabled` | boolean | No | true | Indicates whether fetching issues is enabled. |
129
- | `reviewsEnabled` | boolean | No | true | Indicates whether fetching reviews is enabled. |
130
- | `username` | string | Yes | - | Username on GitHub |
131
- | `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track |
132
- | `filter` | string | Yes | - | Regex to apply to each entry to filter for organization/repo names. For example: `^mozilla` to only include orgs and repos that start with "mozilla". |
133
- | `types` | Object | No | - | Object containing the different labels below |
134
- | `types.commit` | string | No | GitHub Commit | Label for entries related to GitHub commits |
135
- | `types.privateCommit` | string | No | Commit in private repository | Label for entries related to private GitHub commits |
136
- | `types.createdPR` | string | No | Created PR | Label for entries related to opening GitHub PRs |
137
- | `types.createdIssue` | string | No | Created Issue Report | Label for entries related to creating GitHub issues |
138
- | `types.commented` | string | No | Commented on an Issue | Label for entries related to commenting on an issue |
139
- | `types.commentedPR` | string | No | Commented on a Pull Request | Label for entries related to commenting on a GitHub PR (review) |
140
- | `types.approvedPR` | string | No | Approved a Pull Request | Label for entries related to approving a GitHub PR |
141
- | `types.changesRequestedPR` | string | No | Requested changes on a Pull Request | Label for entries related to requesting changes on a GitHub PR |
142
- | `types.reviewedPR` | string | No | Reviewed a Pull Request | Label for entries related to reviewing a GitHub PR (fallback) |
143
- | `delayMsPerRequest` | number | No | 2000 | Delay between each request to GitHub. You might hit rate limiting if you set this lower than default. |
122
+ | Field | Data Type | Required | Default | Description |
123
+ | -------------------------- | --------- | -------- | ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
124
+ | `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `github` object entirely. |
125
+ | `allowPrivate` | boolean | No | false | Will throw if you pass a `GITHUB_TOKEN` with possible private repo scope. In most cases you will not want to have a token with private scope. Please think twice before enabling this config. When using fine-grained tokens, you need to set this to `true` manually as we can't detect the scopes. In that case, we can't guarantee anything! |
126
+ | `commentsEnabled` | boolean | No | true | Indicates whether fetching comments is enabled. |
127
+ | `commitsEnabled` | boolean | No | true | Indicates whether fetching commits is enabled. |
128
+ | `issuesEnabled` | boolean | No | true | Indicates whether fetching issues is enabled. |
129
+ | `reviewsEnabled` | boolean | No | true | Indicates whether fetching reviews is enabled. |
130
+ | `username` | string | Yes | - | Username on GitHub |
131
+ | `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track |
132
+ | `filter` | string | Yes | - | Regex to apply to each entry to filter for organization/repo names. For example: `^mozilla` to only include orgs and repos that start with "mozilla". |
133
+ | `types` | Object | No | - | Object containing the different labels below |
134
+ | `types.commit` | string | No | GitHub Commit | Label for entries related to GitHub commits |
135
+ | `types.privateCommit` | string | No | Commit in private repository | Label for entries related to private GitHub commits |
136
+ | `types.createdPR` | string | No | Created PR | Label for entries related to opening GitHub PRs |
137
+ | `types.createdIssue` | string | No | Created Issue Report | Label for entries related to creating GitHub issues |
138
+ | `types.commented` | string | No | Commented on an Issue | Label for entries related to commenting on an issue |
139
+ | `types.commentedPR` | string | No | Commented on a Pull Request | Label for entries related to commenting on a GitHub PR (review) |
140
+ | `types.approvedPR` | string | No | Approved a Pull Request | Label for entries related to approving a GitHub PR |
141
+ | `types.changesRequestedPR` | string | No | Requested changes on a Pull Request | Label for entries related to requesting changes on a GitHub PR |
142
+ | `types.reviewedPR` | string | No | Reviewed a Pull Request | Label for entries related to reviewing a GitHub PR (fallback) |
143
+ | `delayMsPerRequest` | number | No | 2000 | Delay between each request to GitHub. You might hit rate limiting if you set this lower than default. |
144
144
 
145
145
  #### MediaWiki
146
146
 
@@ -2,9 +2,11 @@ const debug = require('debug')('contributions:github-fetcher');
2
2
  const { Octokit } = require('@octokit/rest');
3
3
 
4
4
  const RESULTS_PER_PAGE = 100;
5
+ const PRIVATE_SCOPE_ERROR = 'PRIVATE_SCOPE';
5
6
 
6
7
  module.exports = {
7
8
  RESULTS_PER_PAGE,
9
+ PRIVATE_SCOPE_ERROR,
8
10
  searchCommits,
9
11
  searchIssues,
10
12
  listReviews,
@@ -76,8 +78,16 @@ async function listComments(options) {
76
78
  async function throwIfTokenHasPrivateRepoScope() {
77
79
  const octokit = getOctokit();
78
80
  const response = await octokit.rest.meta.root();
81
+ // If we don't have the scopes, such as for fine-grained
82
+ // access tokens, we error as well. We can't guarantee
83
+ // that the passed token is not indeed allowing private
84
+ // access.
85
+ if (!response.headers['x-oauth-scopes']) {
86
+ throw new Error(PRIVATE_SCOPE_ERROR);
87
+ }
88
+
79
89
  const scopes = response.headers['x-oauth-scopes'].split(', ');
80
90
  if (scopes.includes('repo')) {
81
- throw new Error('PRIVATE_SCOPE');
91
+ throw new Error(PRIVATE_SCOPE_ERROR);
82
92
  }
83
93
  }
package/lib/github.js CHANGED
@@ -54,12 +54,12 @@ async function gather({ github }, existingContributions = []) {
54
54
  const latestCommit = existingContributions.find((contribution) => {
55
55
  return contribution.source === GITHUB_COMMIT_CATEGORY;
56
56
  }) || { createdAt: github.stopDate };
57
- const latestIssue = existingContributions.find((contribution) => {
58
- return (
59
- contribution.source === GITHUB_ISSUES_CREATED_CATEGORY ||
60
- contribution.source === GITHUB_PR_CREATED_CATEGORY
61
- );
62
- }) || { createdAt: github.stopDate };
57
+ const latestIssue = existingContributions.find(
58
+ (contribution) => contribution.source === GITHUB_ISSUES_CREATED_CATEGORY
59
+ ) || { createdAt: github.stopDate };
60
+ const latestPR = existingContributions.find(
61
+ (contribution) => contribution.source === GITHUB_PR_CREATED_CATEGORY
62
+ ) || { createdAt: github.stopDate };
63
63
  const latestReview = existingContributions.find((contribution) => {
64
64
  return contribution.source === GITHUB_REVIEW_CATEGORY;
65
65
  }) || { createdAt: github.stopDate };
@@ -70,10 +70,10 @@ async function gather({ github }, existingContributions = []) {
70
70
  try {
71
71
  await fetcher.throwIfTokenHasPrivateRepoScope();
72
72
  } catch (error) {
73
- if (error.message === 'PRIVATE_SCOPE') {
73
+ if (error.message === fetcher.PRIVATE_SCOPE_ERROR) {
74
74
  if (!github.allowPrivate) {
75
75
  throw new Error(
76
- 'GITHUB_TOKEN has private repo scope, but the config does not specify allowPrivate to fetch private repos. Are you sure you want to fetch private information?'
76
+ 'GITHUB_TOKEN might have private repo scope, but the config does not specify `allowPrivate` to fetch private repos. Are you sure you want to fetch private information? This might also be because you passed a fine-grained token. In that case we can not check the permissions it has and we assume it is private. You can double check the permissions and if you are sure it is correct, you may set `allowPrivate` to true.'
77
77
  );
78
78
  }
79
79
  } else {
@@ -83,9 +83,16 @@ async function gather({ github }, existingContributions = []) {
83
83
 
84
84
  const commits = await processCommits(github, latestCommit.createdAt);
85
85
  const issues = await processIssues(github, latestIssue.createdAt);
86
+ const prs = await processPRs(github, latestPR.createdAt);
86
87
  const reviews = await processReviews(github, latestReview.createdAt);
87
88
  const comments = await processComments(github, latestComment.createdAt);
88
- const contributions = [...commits, ...issues, ...reviews, ...comments];
89
+ const contributions = [
90
+ ...commits,
91
+ ...issues,
92
+ ...prs,
93
+ ...reviews,
94
+ ...comments,
95
+ ];
89
96
 
90
97
  debug('Finished gathering contributions');
91
98
  return contributions;
@@ -120,6 +127,35 @@ async function processIssues(github, latestCreationDate) {
120
127
  });
121
128
  }
122
129
 
130
+ async function processPRs(github, latestCreationDate) {
131
+ if (
132
+ typeof github.issuesEnabled !== 'undefined' &&
133
+ github.issuesEnabled === false
134
+ ) {
135
+ debug('Fetching PRs is disabled through `issuesEnabled`, skipping!');
136
+ return [];
137
+ }
138
+
139
+ debug('Getting PRs from Github');
140
+ const allPRs = [];
141
+
142
+ const lowerBoundDate = new Date(latestCreationDate);
143
+ const generatedPeriod = periodGenerator(github.stopDate, lowerBoundDate);
144
+ for await (const periodChunk of generatedPeriod) {
145
+ const currentPage = pageGenerator(github, periodChunk, (period, page) =>
146
+ getPRPage(github, 'author', period, page)
147
+ );
148
+ for await (const prs of currentPage) {
149
+ debug(`Got ${prs.length} PRs`);
150
+ allPRs.push(prs);
151
+ }
152
+ }
153
+
154
+ return allPRs.flat().filter((issue) => {
155
+ return new Date(issue.createdAt) > new Date(latestCreationDate);
156
+ });
157
+ }
158
+
123
159
  async function processCommits(github, latestCreationDate) {
124
160
  if (
125
161
  typeof github.commitsEnabled !== 'undefined' &&
@@ -173,7 +209,7 @@ async function processReviews(github, latestCreationDate) {
173
209
  const currentPage = pageGenerator(
174
210
  github,
175
211
  periodChunk,
176
- (period, page) => getPRPage(github, period, page),
212
+ (period, page) => getPRPage(github, 'reviewed-by', period, page),
177
213
  { skipProcessing: true }
178
214
  );
179
215
  for await (const prs of currentPage) {
@@ -365,9 +401,8 @@ async function getIssuePage(github, relationship, issuesPeriod, page) {
365
401
  debug(`Getting issues page ${page} between ${after} and ${before}`);
366
402
  await waitFor(github.delayMsPerRequest ?? DELAY_FETCHING_MS);
367
403
  try {
368
- const additionalFields = relationship === 'commenter' ? '+is:issue' : '';
369
404
  const items = await fetcher.searchIssues({
370
- q: `${relationship}:${github.username}+created:${after}..${before}${additionalFields}`,
405
+ q: `${relationship}:${github.username}+created:${after}..${before}+is:issue`,
371
406
  sort: 'created',
372
407
  page,
373
408
  });
@@ -383,14 +418,14 @@ async function getIssuePage(github, relationship, issuesPeriod, page) {
383
418
  }
384
419
  }
385
420
 
386
- async function getPRPage(github, reviewPeriod, page) {
421
+ async function getPRPage(github, relationship, reviewPeriod, page) {
387
422
  const [before, after] = reviewPeriod;
388
423
 
389
- debug(`Getting reviews page ${page} between ${after} and ${before}`);
424
+ debug(`Getting PR page ${page} between ${after} and ${before}`);
390
425
  await waitFor(github.delayMsPerRequest ?? DELAY_FETCHING_MS);
391
426
  try {
392
427
  const items = await fetcher.searchIssues({
393
- q: `reviewed-by:${github.username}+created:${after}..${before}`,
428
+ q: `${relationship}:${github.username}+created:${after}..${before}+is:pull-request`,
394
429
  sort: 'created',
395
430
  page,
396
431
  });
@@ -407,22 +407,22 @@ test.serial('should fetch issues until stop date', async (t) => {
407
407
 
408
408
  await github.gather(commitConfig);
409
409
  t.deepEqual(fetcher.searchIssues.getCall(0).args[0], {
410
- q: `author:${config.github.username}+created:2022-03-01..2022-03-31`,
410
+ q: `author:${config.github.username}+created:2022-03-01..2022-03-31+is:issue`,
411
411
  sort: 'created',
412
412
  page: 1,
413
413
  });
414
414
  t.deepEqual(fetcher.searchIssues.getCall(1).args[0], {
415
- q: `author:${config.github.username}+created:2022-03-01..2022-03-31`,
415
+ q: `author:${config.github.username}+created:2022-03-01..2022-03-31+is:issue`,
416
416
  sort: 'created',
417
417
  page: 2,
418
418
  });
419
419
  t.deepEqual(fetcher.searchIssues.getCall(2).args[0], {
420
- q: `author:${config.github.username}+created:2022-02-01..2022-02-28`,
420
+ q: `author:${config.github.username}+created:2022-02-01..2022-02-28+is:issue`,
421
421
  sort: 'created',
422
422
  page: 1,
423
423
  });
424
424
  t.deepEqual(fetcher.searchIssues.getCall(3).args[0], {
425
- q: `author:${config.github.username}+created:2022-01-01..2022-01-31`,
425
+ q: `author:${config.github.username}+created:2022-01-01..2022-01-31+is:issue`,
426
426
  sort: 'created',
427
427
  page: 1,
428
428
  });
@@ -611,6 +611,74 @@ test.serial('should ignore issues not matching filter', async (t) => {
611
611
  t.is(result.length, 0);
612
612
  });
613
613
 
614
+ test.serial('should fetch PRs until stop date', async (t) => {
615
+ const commitConfig = {
616
+ github: {
617
+ ...config.github,
618
+ commitsEnabled: false,
619
+ reviewsEnabled: false,
620
+ },
621
+ };
622
+
623
+ fetcher.searchIssues.resolves([]);
624
+ // We want to fetch 2 pages for the first period, and one page for the subsequent periods.
625
+ // We expect 3 periods to be queried based on the passed stopDate and the faked clock.
626
+ // Therefore PRs start with 3 as before it searches for issues
627
+ fetcher.searchIssues
628
+ .onCall(3)
629
+ .resolves(
630
+ new Array(100).fill(0).map(() => ({ createdAt: new Date('2022-03-01') }))
631
+ );
632
+ fetcher.searchIssues
633
+ .onCall(4)
634
+ .resolves(
635
+ new Array(5).fill(0).map(() => ({ createdAt: new Date('2022-02-01') }))
636
+ );
637
+ fetcher.searchIssues
638
+ .onCall(5)
639
+ .resolves(
640
+ new Array(5).fill(0).map(() => ({ createdAt: new Date('2022-01-01') }))
641
+ );
642
+
643
+ await github.gather(commitConfig);
644
+ t.deepEqual(fetcher.searchIssues.getCall(3).args[0], {
645
+ q: `author:${config.github.username}+created:2022-03-01..2022-03-31+is:pull-request`,
646
+ sort: 'created',
647
+ page: 1,
648
+ });
649
+ t.deepEqual(fetcher.searchIssues.getCall(4).args[0], {
650
+ q: `author:${config.github.username}+created:2022-03-01..2022-03-31+is:pull-request`,
651
+ sort: 'created',
652
+ page: 2,
653
+ });
654
+ t.deepEqual(fetcher.searchIssues.getCall(5).args[0], {
655
+ q: `author:${config.github.username}+created:2022-02-01..2022-02-28+is:pull-request`,
656
+ sort: 'created',
657
+ page: 1,
658
+ });
659
+ t.deepEqual(fetcher.searchIssues.getCall(6).args[0], {
660
+ q: `author:${config.github.username}+created:2022-01-01..2022-01-31+is:pull-request`,
661
+ sort: 'created',
662
+ page: 1,
663
+ });
664
+ });
665
+
666
+ test.serial('should not fetch PRs if disabled', async (t) => {
667
+ const commitConfig = {
668
+ github: {
669
+ ...config.github,
670
+ commitsEnabled: false,
671
+ issuesEnabled: false,
672
+ reviewsEnabled: false,
673
+ filter: 'contribution-test',
674
+ stopDate: new Date('2022-03-01'),
675
+ },
676
+ };
677
+
678
+ await github.gather(commitConfig);
679
+ t.is(fetcher.searchCommits.callCount, 0);
680
+ });
681
+
614
682
  test.serial('should fetch reviews until stop date', async (t) => {
615
683
  const commitConfig = {
616
684
  github: {
@@ -666,22 +734,22 @@ test.serial('should fetch reviews until stop date', async (t) => {
666
734
 
667
735
  await github.gather(commitConfig);
668
736
  t.deepEqual(fetcher.searchIssues.getCall(0).args[0], {
669
- q: `reviewed-by:${config.github.username}+created:2022-03-01..2022-03-31`,
737
+ q: `reviewed-by:${config.github.username}+created:2022-03-01..2022-03-31+is:pull-request`,
670
738
  sort: 'created',
671
739
  page: 1,
672
740
  });
673
741
  t.deepEqual(fetcher.searchIssues.getCall(1).args[0], {
674
- q: `reviewed-by:${config.github.username}+created:2022-03-01..2022-03-31`,
742
+ q: `reviewed-by:${config.github.username}+created:2022-03-01..2022-03-31+is:pull-request`,
675
743
  sort: 'created',
676
744
  page: 2,
677
745
  });
678
746
  t.deepEqual(fetcher.searchIssues.getCall(2).args[0], {
679
- q: `reviewed-by:${config.github.username}+created:2022-02-01..2022-02-28`,
747
+ q: `reviewed-by:${config.github.username}+created:2022-02-01..2022-02-28+is:pull-request`,
680
748
  sort: 'created',
681
749
  page: 1,
682
750
  });
683
751
  t.deepEqual(fetcher.searchIssues.getCall(3).args[0], {
684
- q: `reviewed-by:${config.github.username}+created:2022-01-01..2022-01-31`,
752
+ q: `reviewed-by:${config.github.username}+created:2022-01-01..2022-01-31+is:pull-request`,
685
753
  sort: 'created',
686
754
  page: 1,
687
755
  });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "volunteer-contributions-fetch",
3
- "version": "2.3.3",
3
+ "version": "2.3.5",
4
4
  "description": "Library to fetch volunteer contributions from several sources such as GitHub",
5
5
  "author": "Michael Kohler",
6
6
  "license": "MIT",