volunteer-contributions-fetch 2.1.2 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -9,7 +9,8 @@ This library fetches contributions from several different sources and saves them
9
9
  Currently the following sources are supported. Feel free to contribute a PR if you'd like to support another source!
10
10
 
11
11
  * GitHub
12
- * Issues
12
+ * Issues created
13
+ * Comments on issues
13
14
  * PRs
14
15
  * PR Reviews
15
16
  * MediaWiki
@@ -27,8 +28,6 @@ Each of the sources can be enabled separately. See the configuration section bel
27
28
 
28
29
  The following sources are planned:
29
30
 
30
- * GitHub
31
- * Comments
32
31
  * Bugzilla
33
32
  * Bug reported
34
33
  * Bug commented on
@@ -38,7 +37,7 @@ The following sources are planned:
38
37
 
39
38
  ### Possible future sources
40
39
 
41
- The following sources would be interesting for (at least) Mozilla contributors:
40
+ The following sources would be interesting for (at least) Mozilla contributors, however they do not expose an activity API endpoint to easily gather information for a certain user.
42
41
 
43
42
  * support.mozilla.org
44
43
  * Pontoon
@@ -111,6 +110,7 @@ This section documents all the possible configuration values. For examples see t
111
110
  |----------------------------|-----------|----------|---------|----------------------------------------------------------------------------------------------------------------------------|
112
111
  | `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `github` object entirely. |
113
112
  | `allowPrivate` | boolean | No | false | Indicates whether fetching from private repositories is allowed. If not set, the script will throw if you pass a `GITHUB_TOKEN` with private repo scope. In most cases you will not want to have a token with private scope as you do not want to leak any private information in your contribution overview. Please think twice before enabling this config. |
113
+ | `commentsEnabled` | boolean | No | true | Indicates whether fetching comments is enabled. |
114
114
  | `commitsEnabled` | boolean | No | true | Indicates whether fetching commits is enabled. |
115
115
  | `issuesEnabled` | boolean | No | true | Indicates whether fetching issues is enabled. |
116
116
  | `reviewsEnabled` | boolean | No | true | Indicates whether fetching reviews is enabled. |
@@ -122,6 +122,7 @@ This section documents all the possible configuration values. For examples see t
122
122
  | `types.privateCommit` | string | No | Commit in private repository | Label for entries related to private GitHub commits |
123
123
  | `types.createdPR` | string | No | Created PR | Label for entries related to opening GitHub PRs |
124
124
  | `types.createdIssue` | string | No | Created Issue Report | Label for entries related to creating GitHub issues |
125
+ | `types.commented` | string | No | Commented on an Issue | Label for entries related to commenting on an issue |
125
126
  | `types.commentedPR` | string | No | Commented on a Pull Request | Label for entries related to commenting on a GitHub PR (review) |
126
127
  | `types.approvedPR` | string | No | Approved a Pull Request | Label for entries related to approving a GitHub PR |
127
128
  | `types.changesRequestedPR` | string | No | Requested changes on a Pull Request | Label for entries related to requesting changes on a GitHub PR |
@@ -8,6 +8,7 @@ module.exports = {
8
8
  searchCommits,
9
9
  searchIssues,
10
10
  listReviews,
11
+ listComments,
11
12
  throwIfTokenHasPrivateRepoScope,
12
13
  };
13
14
 
@@ -62,6 +63,16 @@ async function listReviews(options) {
62
63
  return response.data;
63
64
  }
64
65
 
66
+ async function listComments(options) {
67
+ const octokit = getOctokit();
68
+ const octokitOptions = {
69
+ ...options,
70
+ per_page: RESULTS_PER_PAGE,
71
+ };
72
+ const response = await octokit.issues.listComments(octokitOptions);
73
+ return response.data;
74
+ }
75
+
65
76
  async function throwIfTokenHasPrivateRepoScope() {
66
77
  const octokit = getOctokit();
67
78
  const response = await octokit.rest.meta.root();
package/lib/github.js CHANGED
@@ -7,6 +7,7 @@ const GITHUB_COMMIT_CATEGORY = 'github-commit';
7
7
  const GITHUB_ISSUES_CREATED_CATEGORY = 'github-issues-created';
8
8
  const GITHUB_PR_CREATED_CATEGORY = 'github-pr-created';
9
9
  const GITHUB_REVIEW_CATEGORY = 'github-reviews';
10
+ const GITHUB_COMMENT_CATEGORY = 'github-comments';
10
11
 
11
12
  const TYPE_COMMIT = 'GitHub Commit';
12
13
  const TYPE_PRIVATE_COMMIT = 'Commit in private repository';
@@ -16,6 +17,7 @@ const TYPE_COMMENTED_PR = 'Commented on a Pull Request';
16
17
  const TYPE_APPROVED_PR = 'Approved a Pull Request';
17
18
  const TYPE_CHANGES_REQUESTED_PR = 'Requested changes on a Pull Request';
18
19
  const TYPE_REVIEWED_PR = 'Reviewed a Pull Request';
20
+ const TYPE_COMMENTED = 'Commented on an Issue';
19
21
 
20
22
  module.exports = {
21
23
  gather,
@@ -58,6 +60,9 @@ async function gather({ github }, existingContributions = []) {
58
60
  const latestReview = existingContributions.find((contribution) => {
59
61
  return contribution.source === GITHUB_REVIEW_CATEGORY;
60
62
  }) || { createdAt: github.stopDate };
63
+ const latestComment = existingContributions.find((contribution) => {
64
+ return contribution.source === GITHUB_COMMENT_CATEGORY;
65
+ }) || { createdAt: github.stopDate };
61
66
 
62
67
  try {
63
68
  await fetcher.throwIfTokenHasPrivateRepoScope();
@@ -74,10 +79,12 @@ async function gather({ github }, existingContributions = []) {
74
79
  const commits = await processCommits(github, latestCommit.createdAt);
75
80
  const issues = await processIssues(github, latestIssue.createdAt);
76
81
  const reviews = await processReviews(github, latestReview.createdAt);
82
+ const comments = await processComments(github, latestComment.createdAt);
77
83
  const contributions = [
78
84
  ...commits,
79
85
  ...issues,
80
86
  ...reviews,
87
+ ...comments,
81
88
  ];
82
89
 
83
90
  debug('Finished gathering contributions');
@@ -99,7 +106,7 @@ async function processIssues(github, latestCreationDate) {
99
106
  const currentPage = pageGenerator(
100
107
  github,
101
108
  periodChunk,
102
- (period, page) => getIssuePage(github, period, page)
109
+ (period, page) => getIssuePage(github, 'author', period, page)
103
110
  );
104
111
  for await (const issues of currentPage) {
105
112
  debug(`Got ${issues.length} issues`);
@@ -187,6 +194,57 @@ async function processReviews(github, latestCreationDate) {
187
194
  });
188
195
  }
189
196
 
197
+ async function processComments(github, latestCreationDate) {
198
+ if (typeof github.commentsEnabled !== 'undefined' && github.commentsEnabled === false) {
199
+ debug('Fetching comments is disabled, skipping!');
200
+ return [];
201
+ }
202
+
203
+ debug('Getting comments from Github');
204
+ const allCommentedIssues = [];
205
+
206
+ // We can only search for commented issues and then get the comments from there. As we
207
+ // can't sort by comment date, we need to fetch a certain number of issues.
208
+ // There is a possibility that an issue was created before `latestCreationDate`
209
+ // but has later been commented on. Therefore we fetch the commented issues for one year
210
+ // and then later on filter out already stored comments.
211
+ const lowerBoundDate = new Date(latestCreationDate) - ONE_YEAR_IN_MS;
212
+ const generatedPeriod = periodGenerator(github.stopDate, lowerBoundDate);
213
+ for await (const periodChunk of generatedPeriod) {
214
+ const currentPage = pageGenerator(
215
+ github,
216
+ periodChunk,
217
+ (period, page) => getIssuePage(github, 'commenter', period, page),
218
+ { skipProcessing: true }
219
+ );
220
+ for await (const commentIssues of currentPage) {
221
+ debug(`Got ${commentIssues.length} commented issues`);
222
+ allCommentedIssues.push(commentIssues);
223
+ }
224
+ }
225
+
226
+ const commentTitleMap = new Map();
227
+ allCommentedIssues.flat()
228
+ .forEach((commentIssue) => commentTitleMap.set(commentIssue.url, commentIssue.title));
229
+
230
+ // Now that we have all the commented issues, we need to fetch the actual comments
231
+ const allComments = [];
232
+
233
+ const commentsList = getComments(
234
+ allCommentedIssues.flat(),
235
+ github.username,
236
+ github.delayMsPerRequest
237
+ );
238
+ for await (const comments of commentsList) {
239
+ allComments.push(comments);
240
+ }
241
+
242
+ const processedComments = processEntities(github, allComments.flat(), commentTitleMap);
243
+ return processedComments.filter((comment) => {
244
+ return new Date(comment.createdAt) > new Date(latestCreationDate);
245
+ });
246
+ }
247
+
190
248
  function getNextPeriod(stopDate, period, lowerBoundDate) {
191
249
  const [before, after] = period;
192
250
 
@@ -278,19 +336,24 @@ async function getCommitPage({ username, delayMsPerRequest }, commitsPeriod, pag
278
336
  }
279
337
  }
280
338
 
281
- async function getIssuePage({ username, delayMsPerRequest }, issuesPeriod, page) {
339
+ async function getIssuePage(github, relationship, issuesPeriod, page) {
282
340
  const [before, after] = issuesPeriod;
283
341
 
284
342
  debug(`Getting issues page ${page} between ${after} and ${before}`);
285
- await waitFor(delayMsPerRequest ?? DELAY_FETCHING_MS);
343
+ await waitFor(github.delayMsPerRequest ?? DELAY_FETCHING_MS);
286
344
  try {
345
+ const additionalFields = relationship === 'commenter' ? '+is:issue' : '';
287
346
  const items = await fetcher.searchIssues({
288
- q: `author:${username}+created:${after}..${before}`,
347
+ q: `${relationship}:${github.username}+created:${after}..${before}${additionalFields}`,
289
348
  sort: 'created',
290
349
  page,
291
350
  });
292
351
 
293
- return items;
352
+ // We already want to filter here, as otherwise we will fetch many comments
353
+ // that will later be discarded anyway.
354
+ const filteredItems = items.filter((item) => filter(github.filter, item));
355
+
356
+ return filteredItems;
294
357
  } catch (error) {
295
358
  debug('Error fetching issues', error);
296
359
  return [];
@@ -347,6 +410,32 @@ async function* getPRReviews(prList, delay) {
347
410
  return [];
348
411
  }
349
412
 
413
+ async function* getComments(issueList, commenterUsername, delay) {
414
+ debug('Getting comments');
415
+ for await (const issue of issueList) {
416
+ try {
417
+ const [, owner, repo] = /github\.com\/repos\/(.+)\/(.*)/.exec(issue.repository_url);
418
+ // We're waiting for a bit here, as we've previously hit a secondary GitHub rate limit...
419
+ await waitFor(delay ?? DELAY_FETCHING_MS);
420
+ debug(`Fetching comments for ${owner}/${repo} issue ${issue.number}`);
421
+ const items = await fetcher.listComments({
422
+ owner,
423
+ repo,
424
+ issue_number: issue.number,
425
+ });
426
+
427
+ const onlyMyComments = items.filter((comment) => comment.user.login === commenterUsername);
428
+
429
+ yield onlyMyComments;
430
+ } catch (error) {
431
+ debug('Error fetching comments', error);
432
+ return [];
433
+ }
434
+ }
435
+
436
+ return [];
437
+ }
438
+
350
439
  async function waitFor(time) {
351
440
  return new Promise((resolve) => {
352
441
  setTimeout(resolve, time);
@@ -375,6 +464,10 @@ function format(types, entity, titleMap) {
375
464
  return formatPRReview(types, entity, titleMap);
376
465
  }
377
466
 
467
+ if (entity.issue_url) {
468
+ return formatComment(types, entity, titleMap);
469
+ }
470
+
378
471
  return formatIssue(types, entity);
379
472
  }
380
473
 
@@ -411,6 +504,21 @@ function formatIssue(types, entity) {
411
504
  };
412
505
  }
413
506
 
507
+ function formatComment(types, entity, titleMap) {
508
+ let description = '';
509
+ if (titleMap) {
510
+ description = titleMap.get(entity.issue_url);
511
+ }
512
+
513
+ return {
514
+ createdAt: new Date(entity.created_at),
515
+ description,
516
+ link: entity.html_url,
517
+ type: types?.commented || TYPE_COMMENTED,
518
+ source: GITHUB_COMMENT_CATEGORY,
519
+ };
520
+ }
521
+
414
522
  function formatPRReview(types, entity, titleMap) {
415
523
  const type = (() => {
416
524
  switch (entity.state) {
@@ -26,6 +26,7 @@ test.beforeEach((t) => {
26
26
  t.context.sandbox.stub(fetcher, 'searchCommits');
27
27
  t.context.sandbox.stub(fetcher, 'searchIssues');
28
28
  t.context.sandbox.stub(fetcher, 'listReviews');
29
+ t.context.sandbox.stub(fetcher, 'listComments');
29
30
  t.context.sandbox.stub(fetcher, 'throwIfTokenHasPrivateRepoScope');
30
31
  });
31
32
 
@@ -820,3 +821,248 @@ test.serial('should ignore reviews not matching filter', async (t) => {
820
821
  const result = await github.gather(commitConfig);
821
822
  t.is(result.length, 0);
822
823
  });
824
+
825
+ test.serial('should fetch comments until stop date', async (t) => {
826
+ const commitConfig = {
827
+ github: {
828
+ ...config.github,
829
+ commitsEnabled: false,
830
+ issuesEnabled: false,
831
+ reviewsEnabled: false,
832
+ },
833
+ };
834
+
835
+ // We want to fetch 2 pages for the first period, and one page for the subsequent periods.
836
+ // We expect 3 periods to be queried based on the passed stopDate and the faked clock.
837
+ fetcher.searchIssues.resolves([]);
838
+ fetcher.searchIssues.onCall(0).resolves(new Array(100).fill(0).map(() => ({
839
+ createdAt: new Date('2022-03-01'),
840
+ repository_url: 'https://api.github.com/repos/contribution-test/foo',
841
+ number: 1111, // This would be dynamic, but fine enough for this test
842
+ title: 'IssueTitle',
843
+ })));
844
+ fetcher.searchIssues.onCall(1).resolves(new Array(5).fill(0).map(() => ({
845
+ createdAt: new Date('2022-02-01'),
846
+ repository_url: 'https://api.github.com/repos/contribution-test/foo',
847
+ number: 1111,
848
+ title: 'IssueTitle',
849
+ })));
850
+ fetcher.searchIssues.onCall(2).resolves(new Array(5).fill(0).map(() => ({
851
+ createdAt: new Date('2022-01-01'),
852
+ repository_url: 'https://api.github.com/repos/contribution-test/foo',
853
+ number: 1111,
854
+ title: 'IssueTitle',
855
+ })));
856
+
857
+ fetcher.listComments.resolves([]);
858
+ fetcher.listComments.onCall(0).resolves([{
859
+ createdAt: new Date('2022-03-01'),
860
+ user: {
861
+ login: config.github.username,
862
+ },
863
+ }]);
864
+
865
+ await github.gather(commitConfig);
866
+ t.deepEqual(fetcher.searchIssues.getCall(0).args[0], {
867
+ q: `commenter:${config.github.username}+created:2022-03-01..2022-03-31+is:issue`,
868
+ sort: 'created',
869
+ page: 1,
870
+ });
871
+ t.deepEqual(fetcher.searchIssues.getCall(1).args[0], {
872
+ q: `commenter:${config.github.username}+created:2022-03-01..2022-03-31+is:issue`,
873
+ sort: 'created',
874
+ page: 2,
875
+ });
876
+ t.deepEqual(fetcher.searchIssues.getCall(2).args[0], {
877
+ q: `commenter:${config.github.username}+created:2022-02-01..2022-02-28+is:issue`,
878
+ sort: 'created',
879
+ page: 1,
880
+ });
881
+ t.deepEqual(fetcher.searchIssues.getCall(3).args[0], {
882
+ q: `commenter:${config.github.username}+created:2022-01-01..2022-01-31+is:issue`,
883
+ sort: 'created',
884
+ page: 1,
885
+ });
886
+ t.is(fetcher.listComments.callCount, 110);
887
+ t.deepEqual(fetcher.listComments.getCall(0).args[0], {
888
+ owner: 'contribution-test',
889
+ repo: 'foo',
890
+ issue_number: 1111,
891
+ });
892
+ });
893
+
894
+ test.serial('should not fetch comments if disabled', async (t) => {
895
+ const commitConfig = {
896
+ github: {
897
+ ...config.github,
898
+ commitsEnabled: false,
899
+ issuesEnabled: false,
900
+ reviewsEnabled: false,
901
+ commentsEnabled: false,
902
+ filter: 'contribution-test',
903
+ stopDate: new Date('2022-03-01'),
904
+ },
905
+ };
906
+
907
+ await github.gather(commitConfig);
908
+ t.is(fetcher.searchIssues.callCount, 0);
909
+ });
910
+
911
+ test.serial('should format comments', async (t) => {
912
+ const commitConfig = {
913
+ github: {
914
+ ...config.github,
915
+ issuesEnabled: false,
916
+ commitsEnabled: false,
917
+ reviewsEnabled: false,
918
+ filter: 'contribution-test',
919
+ stopDate: new Date('2022-03-01'),
920
+ },
921
+ };
922
+
923
+ fetcher.searchIssues.onCall(0).resolves([{
924
+ createdAt: new Date('2022-03-02'),
925
+ repository_url: 'https://api.github.com/repos/contribution-test/foo',
926
+ url: 'https://api.github.com/repos/contribution-test/foo/issues/1111',
927
+ html_url: 'https://github.com/contribution-test/foo/issues/1111',
928
+ title: 'Some Issue',
929
+ number: 1111,
930
+ }]);
931
+
932
+ fetcher.listComments.onCall(0).resolves([{
933
+ created_at: new Date('2022-03-02'),
934
+ issue_url: 'https://api.github.com/repos/contribution-test/foo/issues/1111',
935
+ html_url: 'https://github.com/contribution-test/foo/issues/1111',
936
+ user: {
937
+ login: config.github.username,
938
+ },
939
+ }]);
940
+
941
+ const result = await github.gather(commitConfig);
942
+ t.is(result.length, 1);
943
+ t.deepEqual(result[0], {
944
+ createdAt: new Date('2022-03-02'),
945
+ description: 'Some Issue',
946
+ link: 'https://github.com/contribution-test/foo/issues/1111',
947
+ source: 'github-comments',
948
+ type: 'Commented on an Issue',
949
+ });
950
+ });
951
+
952
+ test.serial('should custom format comments', async (t) => {
953
+ const commitConfig = {
954
+ github: {
955
+ ...config.github,
956
+ issuesEnabled: false,
957
+ commitsEnabled: false,
958
+ reviewsEnabled: false,
959
+ filter: 'contribution-test',
960
+ stopDate: new Date('2022-03-01'),
961
+ types: {
962
+ commented: 'COMMENTED!',
963
+ },
964
+ },
965
+ };
966
+
967
+ fetcher.searchIssues.onCall(0).resolves([{
968
+ createdAt: new Date('2022-03-02'),
969
+ repository_url: 'https://api.github.com/repos/contribution-test/foo',
970
+ url: 'https://api.github.com/repos/contribution-test/foo/issues/1111',
971
+ html_url: 'https://github.com/contribution-test/foo/issues/1111',
972
+ title: 'Some Issue',
973
+ number: 1111,
974
+ }]);
975
+
976
+ fetcher.listComments.onCall(0).resolves([{
977
+ created_at: new Date('2022-03-02'),
978
+ issue_url: 'https://api.github.com/repos/contribution-test/foo/issues/1111',
979
+ html_url: 'https://github.com/contribution-test/foo/issues/1111',
980
+ user: {
981
+ login: config.github.username,
982
+ },
983
+ }]);
984
+
985
+ const result = await github.gather(commitConfig);
986
+ t.is(result.length, 1);
987
+ t.deepEqual(result[0].type, 'COMMENTED!');
988
+ });
989
+
990
+ test.serial('should ignore comments from before stopDate', async (t) => {
991
+ const commitConfig = {
992
+ github: {
993
+ ...config.github,
994
+ issuesEnabled: false,
995
+ commitsEnabled: false,
996
+ reviewsEnabled: false,
997
+ filter: 'contribution-test',
998
+ stopDate: new Date('2022-03-01'),
999
+ },
1000
+ };
1001
+
1002
+ fetcher.searchIssues.onCall(0).resolves([{
1003
+ createdAt: new Date('2022-03-02'),
1004
+ repository_url: 'https://api.github.com/repos/contribution-test/foo',
1005
+ url: 'https://api.github.com/repos/contribution-test/foo/issues/1111',
1006
+ html_url: 'https://github.com/contribution-test/foo/issues/1111',
1007
+ title: 'Some Issue',
1008
+ number: 1111,
1009
+ }]);
1010
+
1011
+ fetcher.listComments.onCall(0).resolves([{
1012
+ created_at: new Date('1999-01-01'),
1013
+ issue_url: 'https://api.github.com/repos/contribution-test/foo/issues/1111',
1014
+ html_url: 'https://github.com/contribution-test/foo/issues/1111',
1015
+ user: {
1016
+ login: config.github.username,
1017
+ },
1018
+ }]);
1019
+
1020
+ const result = await github.gather(commitConfig);
1021
+ t.is(result.length, 0);
1022
+ });
1023
+
1024
+ test.serial('should ignore comments not matching filter', async (t) => {
1025
+ const commitConfig = {
1026
+ github: {
1027
+ ...config.github,
1028
+ issuesEnabled: false,
1029
+ commitsEnabled: false,
1030
+ reviewsEnabled: false,
1031
+ filter: 'contribution-test',
1032
+ stopDate: new Date('2022-03-01'),
1033
+ },
1034
+ };
1035
+
1036
+ fetcher.searchIssues.onCall(0).resolves([{
1037
+ createdAt: new Date('2022-03-02'),
1038
+ repository_url: 'https://api.github.com/repos/not-matching-filter/foo',
1039
+ url: 'https://api.github.com/repos/not-matching-filter/foo/issues/1111',
1040
+ html_url: 'https://github.com/not-matching-filter/foo/issues/1111',
1041
+ title: 'Some Issue',
1042
+ number: 1111,
1043
+ }, {
1044
+ createdAt: new Date('2022-03-02'),
1045
+ repository_url: 'https://api.github.com/repos/contribution-test/foo',
1046
+ url: 'https://api.github.com/repos/contribution-test/foo/issues/1111',
1047
+ html_url: 'https://github.com/contribution-test/foo/issues/1111',
1048
+ title: 'Some Issue',
1049
+ number: 1111,
1050
+ }]);
1051
+
1052
+ // While this is the first call, it will be the call for the second issue
1053
+ // as we filter out the first issue from above due to not matching,.
1054
+ fetcher.listComments.onCall(0).resolves([{
1055
+ created_at: new Date('2022-03-02'),
1056
+ issue_url: 'https://api.github.com/repos/not-matching-filter/foo/issues/1111',
1057
+ html_url: 'https://github.com/not-matching-filter/foo/issues/1111',
1058
+ user: {
1059
+ login: config.github.username,
1060
+ },
1061
+ }]);
1062
+
1063
+ const result = await github.gather(commitConfig);
1064
+ t.is(result.length, 0);
1065
+ // Details should only have been fetched for one issue as the other one is
1066
+ // not matching the filter..
1067
+ t.is(fetcher.listComments.callCount, 1);
1068
+ });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "volunteer-contributions-fetch",
3
- "version": "2.1.2",
3
+ "version": "2.2.0",
4
4
  "description": "Library to fetch volunteer contributions from several sources such as GitHub",
5
5
  "author": "Michael Kohler",
6
6
  "license": "MIT",