volunteer-contributions-fetch 2.1.0 → 2.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -126,6 +126,7 @@ This section documents all the possible configuration values. For examples see t
126
126
  | `types.approvedPR` | string | No | Approved a Pull Request | Label for entries related to approving a GitHub PR |
127
127
  | `types.changesRequestedPR` | string | No | Requested changes on a Pull Request | Label for entries related to requesting changes on a GitHub PR |
128
128
  | `types.reviewedPR` | string | No | Reviewed a Pull Request | Label for entries related to reviewing a GitHub PR (fallback) |
129
+ | `delayMsPerRequest` | number | No | 1000 | Delay between each request to GitHub. You might hit rate limiting if you set this lower than default. |
129
130
 
130
131
  #### MediaWiki
131
132
 
@@ -0,0 +1,259 @@
1
+ const allData = {
2
+ window: {
3
+ document: {
4
+ querySelectorAll: (query) => {
5
+ if (query === '.profile__event') {
6
+ return [{
7
+ querySelector: (singleQuery) => {
8
+ if (singleQuery === '.profile__event-time') {
9
+ return {
10
+ textContent: '2022-08-08',
11
+ };
12
+ }
13
+
14
+ if (singleQuery === '.profile__event-title') {
15
+ return {
16
+ textContent: 'EventA',
17
+ };
18
+ }
19
+
20
+ return {
21
+ textContent: 'we should not have reached here!',
22
+ };
23
+ },
24
+ getAttribute: (key) => {
25
+ if (key === 'href') {
26
+ return 'LinkA';
27
+ }
28
+
29
+ return '';
30
+ },
31
+ }, {
32
+ querySelector: (singleQuery) => {
33
+ if (singleQuery === '.profile__event-time') {
34
+ return {
35
+ textContent: '2022-08-11',
36
+ };
37
+ }
38
+
39
+ if (singleQuery === '.profile__event-title') {
40
+ return {
41
+ textContent: 'EventB',
42
+ };
43
+ }
44
+
45
+ return {
46
+ textContent: 'we should not have reached here!',
47
+ };
48
+ },
49
+ getAttribute: (key) => {
50
+ if (key === 'href') {
51
+ return 'LinkB';
52
+ }
53
+
54
+ return '';
55
+ },
56
+ }];
57
+ }
58
+
59
+ if (query === '.profile__campaign') {
60
+ return [{
61
+ querySelector: (singleQuery) => {
62
+ if (singleQuery === '.profile__campaign-dates') {
63
+ return {
64
+ textContent: '2022-01-01',
65
+ };
66
+ }
67
+
68
+ if (singleQuery === '.profile__campaign-title') {
69
+ return {
70
+ textContent: 'CampaignA',
71
+ };
72
+ }
73
+
74
+ return {
75
+ textContent: 'we should not have reached here!',
76
+ };
77
+ },
78
+ getAttribute: (key) => {
79
+ if (key === 'href') {
80
+ return 'LinkA';
81
+ }
82
+
83
+ return '';
84
+ },
85
+ }, {
86
+ querySelector: (singleQuery) => {
87
+ if (singleQuery === '.profile__campaign-dates') {
88
+ return {
89
+ textContent: '2022-02-01',
90
+ };
91
+ }
92
+
93
+ if (singleQuery === '.profile__campaign-title') {
94
+ return {
95
+ textContent: 'CampaignB',
96
+ };
97
+ }
98
+
99
+ return {
100
+ textContent: 'we should not have reached here!',
101
+ };
102
+ },
103
+ getAttribute: (key) => {
104
+ if (key === 'href') {
105
+ return 'LinkB';
106
+ }
107
+
108
+ return '';
109
+ },
110
+ }];
111
+ }
112
+
113
+ return [];
114
+ },
115
+ },
116
+ },
117
+ };
118
+
119
+ const noData = {
120
+ window: {
121
+ document: {
122
+ querySelectorAll: (query) => {
123
+ if (query === '.profile__event') {
124
+ return [];
125
+ }
126
+
127
+ if (query === '.profile__campaign') {
128
+ return [];
129
+ }
130
+
131
+ return [];
132
+ },
133
+ },
134
+ },
135
+ };
136
+
137
+ const futureData = {
138
+ window: {
139
+ document: {
140
+ querySelectorAll: (query) => {
141
+ if (query === '.profile__event') {
142
+ return [{
143
+ querySelector: (singleQuery) => {
144
+ if (singleQuery === '.profile__event-time') {
145
+ return {
146
+ textContent: '2999-08-08',
147
+ };
148
+ }
149
+
150
+ if (singleQuery === '.profile__event-title') {
151
+ return {
152
+ textContent: 'FutureEvent',
153
+ };
154
+ }
155
+
156
+ return {
157
+ textContent: 'we should not have reached here!',
158
+ };
159
+ },
160
+ getAttribute: (key) => {
161
+ if (key === 'href') {
162
+ return 'LinkA';
163
+ }
164
+
165
+ return '';
166
+ },
167
+ }, {
168
+ querySelector: (singleQuery) => {
169
+ if (singleQuery === '.profile__event-time') {
170
+ return {
171
+ textContent: '2022-08-11',
172
+ };
173
+ }
174
+
175
+ if (singleQuery === '.profile__event-title') {
176
+ return {
177
+ textContent: 'PastEvent',
178
+ };
179
+ }
180
+
181
+ return {
182
+ textContent: 'we should not have reached here!',
183
+ };
184
+ },
185
+ getAttribute: (key) => {
186
+ if (key === 'href') {
187
+ return 'LinkB';
188
+ }
189
+
190
+ return '';
191
+ },
192
+ }];
193
+ }
194
+
195
+ if (query === '.profile__campaign') {
196
+ return [{
197
+ querySelector: (singleQuery) => {
198
+ if (singleQuery === '.profile__campaign-dates') {
199
+ return {
200
+ textContent: '2999-01-01',
201
+ };
202
+ }
203
+
204
+ if (singleQuery === '.profile__campaign-title') {
205
+ return {
206
+ textContent: 'FutureCampaign',
207
+ };
208
+ }
209
+
210
+ return {
211
+ textContent: 'we should not have reached here!',
212
+ };
213
+ },
214
+ getAttribute: (key) => {
215
+ if (key === 'href') {
216
+ return 'LinkA';
217
+ }
218
+
219
+ return '';
220
+ },
221
+ }, {
222
+ querySelector: (singleQuery) => {
223
+ if (singleQuery === '.profile__campaign-dates') {
224
+ return {
225
+ textContent: '2022-02-01',
226
+ };
227
+ }
228
+
229
+ if (singleQuery === '.profile__campaign-title') {
230
+ return {
231
+ textContent: 'PastCampaign',
232
+ };
233
+ }
234
+
235
+ return {
236
+ textContent: 'we should not have reached here!',
237
+ };
238
+ },
239
+ getAttribute: (key) => {
240
+ if (key === 'href') {
241
+ return 'LinkB';
242
+ }
243
+
244
+ return '';
245
+ },
246
+ }];
247
+ }
248
+
249
+ return [];
250
+ },
251
+ },
252
+ },
253
+ };
254
+
255
+ module.exports = {
256
+ allData,
257
+ noData,
258
+ futureData,
259
+ };
@@ -0,0 +1,57 @@
1
+ const firstMonth = `
2
+ <rss version="2.0">
3
+ <channel>
4
+ <title>MozillaWiki - User contributions [en]</title>
5
+ <link>https://wiki.mozilla.org/Special:Contributions/User</link>
6
+ <description>User contributions</description>
7
+ <language>en</language>
8
+ <generator>MediaWiki 1.27.4</generator>
9
+ <lastBuildDate>Fri, 26 Aug 2022 20:13:05 GMT</lastBuildDate>
10
+ <item>
11
+ <title>PageA</title>
12
+ <link>LinkA</link>
13
+ <guid isPermaLink="false">GUID</guid>
14
+ <description>Blabla</description>
15
+ <pubDate>Sat, 01 Jan 2022 00:00:00 GMT</pubDate>
16
+ <dc:creator>User</dc:creator>
17
+ <comments>https://wiki.mozilla.org/Talk:PageA</comments>
18
+ </item>
19
+ <item>
20
+ <title>PageB</title>
21
+ <link>LinkB</link>
22
+ <guid isPermaLink="false">GUID</guid>
23
+ <description>Blabla</description>
24
+ <pubDate>Sun, 02 Jan 2022 00:00:00 GMT</pubDate>
25
+ <dc:creator>User</dc:creator>
26
+ <comments>https://wiki.mozilla.org/Talk:PageA</comments>
27
+ </item>
28
+ </channel>
29
+ </rss>
30
+ `;
31
+
32
+ const secondMonth = `
33
+ <rss version="2.0">
34
+ <channel>
35
+ <title>MozillaWiki - User contributions [en]</title>
36
+ <link>https://wiki.mozilla.org/Special:Contributions/User</link>
37
+ <description>User contributions</description>
38
+ <language>en</language>
39
+ <generator>MediaWiki 1.27.4</generator>
40
+ <lastBuildDate>Fri, 26 Aug 2022 20:13:05 GMT</lastBuildDate>
41
+ <item>
42
+ <title>PageC</title>
43
+ <link>LinkC</link>
44
+ <guid isPermaLink="false">GUID</guid>
45
+ <description>Blabla</description>
46
+ <pubDate>Wed, 01 Dec 2021 00:00:00 GMT</pubDate>
47
+ <dc:creator>User</dc:creator>
48
+ <comments>https://wiki.mozilla.org/Talk:PageC</comments>
49
+ </item>
50
+ </channel>
51
+ </rss>
52
+ `;
53
+
54
+ module.exports = {
55
+ firstMonth,
56
+ secondMonth,
57
+ };
@@ -0,0 +1,149 @@
1
+ const test = require('ava');
2
+ const { JSDOM } = require('jsdom');
3
+ const sinon = require('sinon');
4
+
5
+ const fixtures = require('../fixtures/community-portal');
6
+ const communityPortal = require('./community-portal');
7
+
8
+ const config = {
9
+ communityPortal: {
10
+ enabled: true,
11
+ baseUrl: 'https://community.mozilla.org',
12
+ username: 'user',
13
+ }
14
+ };
15
+
16
+ test.beforeEach((t) => {
17
+ // eslint-disable-next-line no-param-reassign
18
+ t.context.sandbox = sinon.createSandbox();
19
+
20
+ t.context.sandbox.stub(JSDOM, 'fromURL');
21
+ });
22
+
23
+ test.afterEach.always((t) => {
24
+ t.context.sandbox.restore();
25
+ });
26
+
27
+ test.serial('should validate config - missing completely', (t) => {
28
+ const emptyConfig = {};
29
+ t.notThrows(() => communityPortal.validate(emptyConfig));
30
+ });
31
+
32
+ test.serial('should validate config - missing baseUrl', (t) => {
33
+ const failConfig = {
34
+ enabled: true,
35
+ username: 'A',
36
+ };
37
+
38
+ t.throws(() => communityPortal.validate(failConfig));
39
+ });
40
+
41
+ test.serial('should validate config - missing username', (t) => {
42
+ const failConfig = {
43
+ enabled: true,
44
+ baseUrl: 'A',
45
+ };
46
+
47
+ t.throws(() => communityPortal.validate(failConfig));
48
+ });
49
+
50
+ test.serial('should fetch', async (t) => {
51
+ JSDOM.fromURL.resolves(fixtures.allData);
52
+
53
+ const contributions = await communityPortal.gather(config);
54
+ t.is(contributions.length, 4);
55
+ });
56
+
57
+ test.serial('should fetch even though sections are empty', async (t) => {
58
+ JSDOM.fromURL.resolves(fixtures.noData);
59
+
60
+ const contributions = await communityPortal.gather(config);
61
+ t.is(contributions.length, 0);
62
+ });
63
+
64
+ test.serial('should only fetch events and campaigns that are not in the future', async (t) => {
65
+ JSDOM.fromURL.resolves(fixtures.futureData);
66
+
67
+ const contributions = await communityPortal.gather(config);
68
+ t.is(contributions.length, 2);
69
+ t.deepEqual(contributions[0].createdAt, new Date('2022-08-11'));
70
+ t.is(contributions[0].description, 'PastEvent');
71
+ t.deepEqual(contributions[1].createdAt, new Date('2022-02-01'));
72
+ t.is(contributions[1].description, 'PastCampaign');
73
+ });
74
+
75
+ test.serial('should not fetch if disabled - by enable property', async (t) => {
76
+ const specificTypeConfig = {
77
+ communityPortal: {
78
+ ...config.communityPortal,
79
+ enabled: false,
80
+ },
81
+ };
82
+
83
+ const contributions = await communityPortal.gather(specificTypeConfig);
84
+
85
+ t.is(contributions.length, 0);
86
+ });
87
+
88
+ test.serial('should not fetch if disabled - by leaving off config value', async (t) => {
89
+ const contributions = await communityPortal.gather({});
90
+
91
+ t.is(contributions.length, 0);
92
+ });
93
+
94
+ test.serial('should format', async (t) => {
95
+ JSDOM.fromURL.resolves(fixtures.allData);
96
+
97
+ const contributions = await communityPortal.gather(config);
98
+
99
+ // Event
100
+ t.deepEqual(contributions[0], {
101
+ createdAt: new Date('2022-08-11'),
102
+ description: 'EventB',
103
+ link: 'LinkB',
104
+ type: 'Participated in an event',
105
+ source: 'community-portal-events',
106
+ });
107
+ t.deepEqual(contributions[1], {
108
+ createdAt: new Date('2022-08-08'),
109
+ description: 'EventA',
110
+ link: 'LinkA',
111
+ type: 'Participated in an event',
112
+ source: 'community-portal-events',
113
+ });
114
+ // Campaign
115
+ t.deepEqual(contributions[2], {
116
+ createdAt: new Date('2022-02-01'),
117
+ description: 'CampaignB',
118
+ link: 'LinkB',
119
+ type: 'Participated in a campaign',
120
+ source: 'community-portal-campaigns',
121
+ });
122
+ t.deepEqual(contributions[3], {
123
+ createdAt: new Date('2022-01-01'),
124
+ description: 'CampaignA',
125
+ link: 'LinkA',
126
+ type: 'Participated in a campaign',
127
+ source: 'community-portal-campaigns',
128
+ });
129
+ });
130
+
131
+ test.serial('should format with custom types', async (t) => {
132
+ JSDOM.fromURL.resolves(fixtures.allData);
133
+
134
+ const specificTypeConfig = {
135
+ communityPortal: {
136
+ ...config.communityPortal,
137
+ participationType: 'Event!',
138
+ campaignType: 'Campaign!',
139
+ }
140
+ };
141
+
142
+ const contributions = await communityPortal.gather(specificTypeConfig);
143
+
144
+ // Event
145
+ t.is(contributions[0].type, specificTypeConfig.communityPortal.participationType);
146
+ t.is(contributions[1].type, specificTypeConfig.communityPortal.participationType);
147
+ t.is(contributions[2].type, specificTypeConfig.communityPortal.campaignType);
148
+ t.is(contributions[3].type, specificTypeConfig.communityPortal.campaignType);
149
+ });
@@ -1,5 +1,3 @@
1
- /* eslint camelcase: 0 */
2
-
3
1
  const test = require('ava');
4
2
  const axios = require('axios');
5
3
  const sinon = require('sinon');
@@ -8,7 +6,6 @@ const discourse = require('./discourse');
8
6
 
9
7
  const TOPICS = {
10
8
  topic_list: {
11
- // eslint-disable-next-line newline-per-chained-call
12
9
  topics: new Array(30).fill(0).map((_, index) => ({
13
10
  created_at: new Date('2020-04-13'),
14
11
  title: `Topic ${index}`,
@@ -30,7 +27,6 @@ const TOPICS_SMALL = {
30
27
  };
31
28
 
32
29
  const POSTS = {
33
- // eslint-disable-next-line newline-per-chained-call
34
30
  user_actions: new Array(30).fill(0).map((_, index) => ({
35
31
  created_at: new Date('2020-04-13'),
36
32
  title: `Post ${index}`,
@@ -56,9 +52,8 @@ const config = {
56
52
  discourse: {
57
53
  enabled: true,
58
54
  baseUrl: 'https://discourse.mozilla.org',
59
- topicType: 'Created Discourse Topic',
60
- postType: 'Posted on Discourse Topic',
61
- },
55
+ username: 'user',
56
+ }
62
57
  };
63
58
 
64
59
  test.beforeEach((t) => {
@@ -79,11 +74,53 @@ test.afterEach.always((t) => {
79
74
  t.context.sandbox.restore();
80
75
  });
81
76
 
77
+ test.serial('should validate config - missing completely', (t) => {
78
+ const emptyConfig = {};
79
+ t.notThrows(() => discourse.validate(emptyConfig));
80
+ });
81
+
82
+ test.serial('should validate config - missing baseUrl', (t) => {
83
+ const failConfig = {
84
+ enabled: true,
85
+ username: 'A',
86
+ };
87
+
88
+ t.throws(() => discourse.validate(failConfig));
89
+ });
90
+
91
+ test.serial('should validate config - missing username', (t) => {
92
+ const failConfig = {
93
+ enabled: true,
94
+ baseUrl: 'A',
95
+ };
96
+
97
+ t.throws(() => discourse.validate(failConfig));
98
+ });
99
+
82
100
  test.serial('should fetch', async (t) => {
83
101
  const contributions = await discourse.gather(config);
84
102
  t.is(contributions.length, 30 + 1 + 30 + 1);
85
103
  });
86
104
 
105
+ test.serial('should not fetch if disabled - by enable property', async (t) => {
106
+ const specificTypeConfig = {
107
+ discourse: {
108
+ ...config.discourse,
109
+ enabled: false,
110
+ },
111
+ };
112
+
113
+ const contributions = await discourse.gather(specificTypeConfig);
114
+
115
+ t.is(contributions.length, 0);
116
+ });
117
+
118
+ test.serial('should not fetch if disabled - by leaving off config value', async (t) => {
119
+ const contributions = await discourse.gather({});
120
+
121
+ t.is(contributions.length, 0);
122
+ });
123
+
87
124
  test.serial('should format', async (t) => {
88
125
  const contributions = await discourse.gather(config);
89
126
 
@@ -104,3 +141,32 @@ test.serial('should format', async (t) => {
104
141
  source: 'discourse-posts',
105
142
  });
106
143
  });
144
+
145
+ test.serial('should format with custom types', async (t) => {
146
+ const specificTypeConfig = {
147
+ discourse: {
148
+ ...config.discourse,
149
+ topicType: 'Topic created!',
150
+ postType: 'Posted!',
151
+ }
152
+ };
153
+
154
+ const contributions = await discourse.gather(specificTypeConfig);
155
+
156
+ // Topic
157
+ t.deepEqual(contributions[0], {
158
+ createdAt: new Date('2020-04-13'),
159
+ description: 'Topic 0',
160
+ link: 'https://discourse.mozilla.org/t/slug-0/0',
161
+ type: specificTypeConfig.discourse.topicType,
162
+ source: 'discourse-topics',
163
+ });
164
+ // Post
165
+ t.deepEqual(contributions[31], {
166
+ createdAt: new Date('2020-04-13'),
167
+ description: 'Post 0',
168
+ link: 'https://discourse.mozilla.org/t/slug-0/0/1',
169
+ type: specificTypeConfig.discourse.postType,
170
+ source: 'discourse-posts',
171
+ });
172
+ });
package/lib/fetch.js CHANGED
@@ -1,5 +1,5 @@
1
1
  const debug = require('debug')('contributions:gather');
2
- const fs = require('fs').promises;
2
+ const fs = require('fs/promises');
3
3
 
4
4
  const communityPortal = require('./community-portal');
5
5
  const discourse = require('./discourse');
@@ -0,0 +1,72 @@
1
+ const test = require('ava');
2
+ const fs = require('fs/promises');
3
+ const sinon = require('sinon');
4
+
5
+ const communityPortal = require('./community-portal');
6
+ const discourse = require('./discourse');
7
+ const github = require('./github');
8
+ const wiki = require('./wiki');
9
+
10
+ const validator = require('./validator');
11
+ const fetch = require('./fetch');
12
+
13
+ const config = {
14
+ outputFile: 'file',
15
+ };
16
+
17
+ test.beforeEach((t) => {
18
+ // eslint-disable-next-line no-param-reassign
19
+ t.context.sandbox = sinon.createSandbox();
20
+
21
+ // eslint-disable-next-line no-param-reassign
22
+ t.context.sandbox.originalJSONParse = JSON.parse;
23
+ t.context.sandbox.stub(fs, 'readFile').resolves('');
24
+ t.context.sandbox.stub(fs, 'writeFile').resolves('');
25
+ t.context.sandbox.stub(JSON, 'parse').returns([]);
26
+ t.context.sandbox.stub(validator, 'validateConfig').returns([]);
27
+ t.context.sandbox.stub(communityPortal, 'gather').returns([]);
28
+ t.context.sandbox.stub(discourse, 'gather').returns([]);
29
+ t.context.sandbox.stub(github, 'gather').returns([]);
30
+ t.context.sandbox.stub(wiki, 'gather').returns([]);
31
+ });
32
+
33
+ test.afterEach.always((t) => {
34
+ t.context.sandbox.restore();
35
+ });
36
+
37
+ test.serial('should not write file if there was nothing to do', async (t) => {
38
+ const result = await fetch.fetchAll(config);
39
+ t.is(result, undefined);
40
+ t.is(fs.writeFile.callCount, 0);
41
+ });
42
+
43
+ test.serial('should consider saved contributions if not full-fetch source', async (t) => {
44
+ const existingContributions = [{ source: 'keep_me' }];
45
+ JSON.parse.returns(existingContributions);
46
+ github.gather.returns([{ source: 'github' }]);
47
+
48
+ await fetch.fetchAll(config);
49
+ t.is(fs.writeFile.callCount, 1);
50
+ t.is(t.context.sandbox.originalJSONParse(fs.writeFile.getCall(0).args[1]).length, 2);
51
+ });
52
+
53
+ test.serial('should ignore saved contributions if full-fetch source', async (t) => {
54
+ const existingContributions = [{ source: 'wiki' }, { source: 'wiki' }];
55
+ JSON.parse.returns(existingContributions);
56
+ github.gather.returns([{ source: 'github' }]);
57
+
58
+ await fetch.fetchAll(config);
59
+ t.is(fs.writeFile.callCount, 1);
60
+ t.is(t.context.sandbox.originalJSONParse(fs.writeFile.getCall(0).args[1]).length, 1);
61
+ });
62
+
63
+ test.serial('should take all return values', async (t) => {
64
+ communityPortal.gather.returns([{ source: 'community-portal' }]);
65
+ discourse.gather.returns([{ source: 'discourse' }]);
66
+ github.gather.returns([{ source: 'github' }]);
67
+ wiki.gather.returns([{ source: 'wiki' }]);
68
+
69
+ await fetch.fetchAll(config);
70
+ t.is(fs.writeFile.callCount, 1);
71
+ t.is(t.context.sandbox.originalJSONParse(fs.writeFile.getCall(0).args[1]).length, 4);
72
+ });
@@ -36,7 +36,7 @@ async function searchCommits(options) {
36
36
  const octokit = getOctokit();
37
37
  const octokitOptions = {
38
38
  ...options,
39
- per_page: RESULTS_PER_PAGE, // eslint-disable-line camelcase
39
+ per_page: RESULTS_PER_PAGE,
40
40
  };
41
41
  const response = await octokit.search.commits(octokitOptions);
42
42
  return response.data.items;
@@ -46,7 +46,7 @@ async function searchIssues(options) {
46
46
  const octokit = getOctokit();
47
47
  const octokitOptions = {
48
48
  ...options,
49
- per_page: RESULTS_PER_PAGE, // eslint-disable-line camelcase
49
+ per_page: RESULTS_PER_PAGE,
50
50
  };
51
51
  const response = await octokit.search.issuesAndPullRequests(octokitOptions);
52
52
  return response.data.items;
@@ -56,7 +56,7 @@ async function listReviews(options) {
56
56
  const octokit = getOctokit();
57
57
  const octokitOptions = {
58
58
  ...options,
59
- per_page: RESULTS_PER_PAGE, // eslint-disable-line camelcase
59
+ per_page: RESULTS_PER_PAGE,
60
60
  };
61
61
  const response = await octokit.pulls.listReviews(octokitOptions);
62
62
  return response.data;