volunteer-contributions-fetch 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -13,6 +13,9 @@ Currently the following sources are supported. Feel free to contribute a PR if y
13
13
  * Comments on issues
14
14
  * PRs
15
15
  * PR Reviews
16
+ * Bugzilla
17
+ * Bug reported
18
+ * Bug commented on
16
19
  * MediaWiki
17
20
  * Page edited
18
21
  * Discourse
@@ -28,9 +31,6 @@ Each of the sources can be enabled separately. See the configuration section bel
28
31
 
29
32
  The following sources are planned:
30
33
 
31
- * Bugzilla
32
- * Bug reported
33
- * Bug commented on
34
34
  * Phabricator
35
35
  * Revision created
36
36
  * Revision updated
@@ -79,11 +79,24 @@ This section documents all the possible configuration values. For examples see t
79
79
  | Field | Data Type | Required | Default | Description |
80
80
  |-------------------|-----------|----------|---------|----------------------------------------------------|
81
81
  | `outputFile` | string | Yes | - | Path to contributions output file |
82
+ | `bugzilla` | Object | No | - | Bugzilla configuration options (see below) |
82
83
  | `communityPortal` | Object | No | - | Community Portal configuration options (see below) |
83
84
  | `discourse` | Object | No | - | Discourse configuration options (see below) |
84
85
  | `github` | Object | No | - | GitHub configuration options (see below) |
85
86
  | `mediaWiki` | Object | No | - | MediaWiki configuration options (see below) |
86
87
 
88
+ #### Bugzilla
89
+
90
+ | Field | Data Type | Required | Default | Description |
91
+ |-----------------------|-----------|----------|---------------------------|----------------------------------------------------------------------------------------------------------------------------|
92
+ | `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `mediaWiki` object entirely. |
93
+ | `baseUrl` | string | Yes | - | API base url without trailing slash (for example: `https://bugzilla.mozilla.org`) |
94
+ | `username` | string | Yes | - | Username on MediaWiki |
95
+ | `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track |
96
+ | `types` | Object | No | - | Object containing the different labels below |
97
+ | `types.createdType` | string | No | Created a Bug Report | Label for entries related to having report a bug/enhancement. |
98
+ | `types.commentedType` | string | No | Commented on a Bug Report | Label for entries related to having commented on a bug/enhancement. |
99
+
87
100
  #### Community Portal
88
101
 
89
102
  | Field | Data Type | Required | Default | Description |
@@ -144,6 +157,12 @@ This section documents all the possible configuration values. For examples see t
144
157
  ```js
145
158
  const config = {
146
159
  outputFile: `${__dirname}/_data/contributions.json`,
160
+ bugzilla: {
161
+ enabled: true,
162
+ baseUrl: 'https://bugzilla.mozilla.org',
163
+ username: 'me@michaelkohler.info',
164
+ stopDate: '2008-01-01',
165
+ },
147
166
  communityPortal: {
148
167
  enabled: true,
149
168
  baseUrl: 'https://community.mozilla.org/people',
@@ -0,0 +1,141 @@
1
+ const debug = require('debug')('contributions:bugzilla');
2
+ const axios = require('axios');
3
+
4
+ const { getPadded } = require('./utils');
5
+
6
+ const BUGZILLA_CREATED_CATEGORY = 'bugzilla-created';
7
+ const BUGZILLA_COMMENTED_CATEGORY = 'bugzilla-comments';
8
+
9
+ const TYPE_CREATED = 'Created a Bug Report';
10
+ const TYPE_COMMENTED = 'Commented on a Bug Report';
11
+
12
+ const RESULTS_PER_PAGE = 500;
13
+ const FETCH_TIMEOUT = 120000;
14
+
15
+ module.exports = {
16
+ gather,
17
+ validate,
18
+ };
19
+
20
+ function validate(config) {
21
+ debug('Validating Bugzilla config');
22
+
23
+ if (!config) {
24
+ return;
25
+ }
26
+
27
+ if (config && config.enabled && !config.baseUrl) {
28
+ throw new Error('Bugzilla: baseUrl is required');
29
+ }
30
+
31
+ if (config && config.enabled && !config.username) {
32
+ throw new Error('Bugzilla: username is required');
33
+ }
34
+
35
+ if (config && config.enabled && !config.stopDate) {
36
+ throw new Error('Bugzilla: stopDate is required');
37
+ }
38
+ }
39
+
40
+ async function gather({ bugzilla }, existingContributions = []) {
41
+ if (!bugzilla || (bugzilla && !bugzilla.enabled)) {
42
+ debug('Bugzilla source not enabled, skipping');
43
+ return [];
44
+ }
45
+
46
+ const latest = existingContributions.find((contribution) => {
47
+ return contribution.source === BUGZILLA_COMMENTED_CATEGORY
48
+ || contribution.source === BUGZILLA_CREATED_CATEGORY;
49
+ }) || { createdAt: bugzilla.stopDate };
50
+
51
+ const commentedBugs = [];
52
+ const bugsList = bugPageGenerator(bugzilla, latest.createdAt);
53
+ for await (const bugList of bugsList) {
54
+ commentedBugs.push(bugList.map((bug) => ({
55
+ id: bug.id,
56
+ summary: bug.summary,
57
+ })));
58
+ }
59
+ const flatCommentedBugs = commentedBugs.flat();
60
+
61
+ debug(`Got ${flatCommentedBugs.length} bugs this user has commented on.`);
62
+
63
+ const titleMap = new Map();
64
+ flatCommentedBugs.forEach((bug) => titleMap.set(bug.id, bug.summary));
65
+
66
+ // Now that we have all bugs we commented on, we can fetch the comments
67
+ // for each bug..
68
+ let contributions = [];
69
+ const commentsList = getComments(bugzilla, flatCommentedBugs, titleMap);
70
+ for await (const comments of commentsList) {
71
+ contributions = contributions.concat(comments);
72
+ }
73
+
74
+ return contributions;
75
+ }
76
+
77
+ async function* bugPageGenerator(bugzilla, stopDate) {
78
+ let hasMoreToFetch = true;
79
+ let offset = 0;
80
+ const date = new Date(stopDate);
81
+ const stop = `${date.getFullYear()}-${getPadded(date.getMonth() + 1)}-${getPadded(date.getDate())}`;
82
+ while (hasMoreToFetch) {
83
+ const query = `quicksearch=ALL%20commenter:${bugzilla.username}&limit=${RESULTS_PER_PAGE}&offset=${offset}&last_change_time=${stop}`;
84
+ const bugsUrl = `${bugzilla.baseUrl}/rest/bug?${query}`;
85
+ debug(`Fetching from ${bugsUrl}`);
86
+ // eslint-disable-next-line no-await-in-loop
87
+ const { data } = await axios.request({
88
+ method: 'GET',
89
+ url: bugsUrl,
90
+ timeout: FETCH_TIMEOUT,
91
+ });
92
+ debug(`Got ${data.bugs.length} bugs results`);
93
+ // If we have less results than the max result, we're at the end
94
+ if (data.bugs.length < RESULTS_PER_PAGE) {
95
+ hasMoreToFetch = false;
96
+ yield data.bugs;
97
+ return;
98
+ }
99
+
100
+ offset += RESULTS_PER_PAGE;
101
+ yield data.bugs;
102
+ }
103
+ }
104
+
105
+ async function* getComments(bugzilla, bugs, titleMap) {
106
+ for await (const bug of bugs) {
107
+ const commentsUrl = `${bugzilla.baseUrl}/rest/bug/${bug.id}/comment`;
108
+ debug(`Fetching from ${commentsUrl}`);
109
+ // eslint-disable-next-line no-await-in-loop
110
+ const { data } = await axios.request({
111
+ method: 'GET',
112
+ url: commentsUrl,
113
+ timeout: FETCH_TIMEOUT,
114
+ });
115
+
116
+ const ownComments = data.bugs[bug.id].comments
117
+ .filter((comment) => comment.creator === bugzilla.username);
118
+
119
+ yield processEntities(bugzilla, ownComments, titleMap);
120
+ }
121
+
122
+ return [];
123
+ }
124
+
125
+ function processEntities({ types }, entities, titleMap) {
126
+ return entities
127
+ .map((entity) => format(types || {}, entity, titleMap));
128
+ }
129
+
130
+ function format(types, entity, titleMap) {
131
+ const created = types.createdType || TYPE_CREATED;
132
+ const commented = types.commentedType || TYPE_COMMENTED;
133
+
134
+ return {
135
+ createdAt: new Date(entity.creation_time),
136
+ description: titleMap.get(entity.bug_id),
137
+ link: `https://bugzilla.mozilla.org/show_bug.cgi?id=${entity.bug_id}#c${entity.count}`,
138
+ type: entity.count === 0 ? created : commented,
139
+ source: entity.count === 0 ? BUGZILLA_CREATED_CATEGORY : BUGZILLA_COMMENTED_CATEGORY,
140
+ };
141
+ }
@@ -0,0 +1,170 @@
1
+ const test = require('ava');
2
+ const axios = require('axios');
3
+ const sinon = require('sinon');
4
+
5
+ const bugzilla = require('./bugzilla');
6
+
7
+ const config = {
8
+ bugzilla: {
9
+ enabled: true,
10
+ baseUrl: 'https://bugzilla.mozilla.org',
11
+ username: 'user',
12
+ stopDate: new Date('2021-11-01'),
13
+ }
14
+ };
15
+
16
+ test.beforeEach((t) => {
17
+ // eslint-disable-next-line no-param-reassign
18
+ t.context.sandbox = sinon.createSandbox();
19
+
20
+ const firstBugsPage = {
21
+ bugs: new Array(500).fill(0).map(() => ({
22
+ created_at: new Date('2020-04-13'),
23
+ summary: 'Summary',
24
+ // This would be the bug ID which would be different, but enough for testing purposes
25
+ id: 1,
26
+ })),
27
+ };
28
+
29
+ const secondBugsPage = {
30
+ bugs: [{
31
+ created_at: new Date('2020-04-13'),
32
+ summary: 'SecondSummary',
33
+ id: 1,
34
+ }]
35
+ };
36
+
37
+ const comments = {
38
+ bugs: {
39
+ // This would be the bug ID which would be different, but enough for testing purposes
40
+ 1: {
41
+ comments: [{
42
+ creation_time: new Date('2020-04-13'),
43
+ creator: 'user',
44
+ bug_id: 1,
45
+ count: 0,
46
+ }, {
47
+ creation_time: new Date('2020-04-13'),
48
+ creator: 'user',
49
+ bug_id: 1,
50
+ count: 1,
51
+ }, {
52
+ creation_time: new Date('2020-04-13'),
53
+ creator: 'WILL_BE_FILTERED_OUT',
54
+ bug_id: 1,
55
+ count: 2,
56
+ }],
57
+ },
58
+ },
59
+ };
60
+
61
+ t.context.sandbox.stub(axios, 'request');
62
+ axios.request.resolves({ data: comments });
63
+ axios.request.onCall(0)
64
+ .resolves({ data: firstBugsPage });
65
+ axios.request.onCall(1)
66
+ .resolves({ data: secondBugsPage });
67
+ });
68
+
69
+ test.afterEach.always((t) => {
70
+ t.context.sandbox.restore();
71
+ });
72
+
73
+ test.serial('should validate config - undefined', (t) => {
74
+ t.notThrows(() => bugzilla.validate(undefined));
75
+ });
76
+
77
+ test.serial('should validate config - missing completely', (t) => {
78
+ const emptyConfig = {};
79
+ t.notThrows(() => bugzilla.validate(emptyConfig));
80
+ });
81
+
82
+ test.serial('should validate config - missing baseUrl', (t) => {
83
+ const failConfig = {
84
+ enabled: true,
85
+ username: 'A',
86
+ stopDate: new Date(),
87
+ };
88
+
89
+ t.throws(() => bugzilla.validate(failConfig));
90
+ });
91
+
92
+ test.serial('should validate config - missing username', (t) => {
93
+ const failConfig = {
94
+ enabled: true,
95
+ baseUrl: 'A',
96
+ stopDate: new Date(),
97
+ };
98
+
99
+ t.throws(() => bugzilla.validate(failConfig));
100
+ });
101
+
102
+ test.serial('should validate config - missing stopDate', (t) => {
103
+ const failConfig = {
104
+ enabled: true,
105
+ baseUrl: 'A',
106
+ username: 'user',
107
+ };
108
+
109
+ t.throws(() => bugzilla.validate(failConfig));
110
+ });
111
+
112
+ test.serial('should fetch', async (t) => {
113
+ const contributions = await bugzilla.gather(config);
114
+ t.is(contributions.length, (500 + 1) * 2);
115
+ });
116
+
117
+ test.serial('should not fetch if disabled - by enable property', async (t) => {
118
+ const specificTypeConfig = {
119
+ bugzilla: {
120
+ ...config.bugzilla,
121
+ enabled: false,
122
+ },
123
+ };
124
+
125
+ const contributions = await bugzilla.gather(specificTypeConfig);
126
+
127
+ t.is(contributions.length, 0);
128
+ });
129
+
130
+ test.serial('should not fetch if disabled - by leaving off config value', async (t) => {
131
+ const contributions = await bugzilla.gather({});
132
+
133
+ t.is(contributions.length, 0);
134
+ });
135
+
136
+ test.serial('should format', async (t) => {
137
+ const contributions = await bugzilla.gather(config);
138
+
139
+ t.deepEqual(contributions[0], {
140
+ createdAt: new Date('2020-04-13'),
141
+ description: 'SecondSummary',
142
+ link: 'https://bugzilla.mozilla.org/show_bug.cgi?id=1#c0',
143
+ type: 'Created a Bug Report',
144
+ source: 'bugzilla-created',
145
+ });
146
+ t.deepEqual(contributions[1], {
147
+ createdAt: new Date('2020-04-13'),
148
+ description: 'SecondSummary',
149
+ link: 'https://bugzilla.mozilla.org/show_bug.cgi?id=1#c1',
150
+ type: 'Commented on a Bug Report',
151
+ source: 'bugzilla-comments',
152
+ });
153
+ });
154
+
155
+ test.serial('should format with custom types', async (t) => {
156
+ const specificTypeConfig = {
157
+ bugzilla: {
158
+ ...config.bugzilla,
159
+ types: {
160
+ createdType: 'CREATED!',
161
+ commentedType: 'COMMENTED!',
162
+ },
163
+ },
164
+ };
165
+
166
+ const contributions = await bugzilla.gather(specificTypeConfig);
167
+
168
+ t.is(contributions[0].type, specificTypeConfig.bugzilla.types.createdType);
169
+ t.is(contributions[1].type, specificTypeConfig.bugzilla.types.commentedType);
170
+ });
@@ -24,6 +24,10 @@ test.afterEach.always((t) => {
24
24
  t.context.sandbox.restore();
25
25
  });
26
26
 
27
+ test.serial('should validate config - undefined', (t) => {
28
+ t.notThrows(() => communityPortal.validate(undefined));
29
+ });
30
+
27
31
  test.serial('should validate config - missing completely', (t) => {
28
32
  const emptyConfig = {};
29
33
  t.notThrows(() => communityPortal.validate(emptyConfig));
package/lib/fetch.js CHANGED
@@ -1,6 +1,7 @@
1
1
  const debug = require('debug')('contributions:gather');
2
2
  const fs = require('fs/promises');
3
3
 
4
+ const bugzilla = require('./bugzilla');
4
5
  const communityPortal = require('./community-portal');
5
6
  const discourse = require('./discourse');
6
7
  const github = require('./github');
@@ -13,6 +14,7 @@ async function fetchAll(config) {
13
14
  debug('fetching...');
14
15
  const existingContributions = JSON.parse(await fs.readFile(config.outputFile));
15
16
 
17
+ const bugzillaResult = await bugzilla.gather(config, existingContributions);
16
18
  const githubResult = await github.gather(config, existingContributions);
17
19
  const wikiResult = await wiki.gather(config);
18
20
  const communityPortalResult = await communityPortal.gather(config);
@@ -34,6 +36,7 @@ async function fetchAll(config) {
34
36
  });
35
37
  debug('ALREADY_EXISTING_CONTRIBUTIONS', existingContributionsToSave.length);
36
38
 
39
+ debug('NEW_BUGZILLA_CONTRIBUTIONS', bugzillaResult);
37
40
  debug('NEW_GITHUB_CONTRIBUTIONS', githubResult);
38
41
  debug('COMMUNITY_PORTAL_CONTRIBUTIONS', communityPortalResult.length);
39
42
  debug('DISCOURSE_CONTRIBUTIONS', discourseResult.length);
@@ -45,6 +48,7 @@ async function fetchAll(config) {
45
48
  ...wikiResult, // always a new fetch (previous entries removed above)
46
49
  ...communityPortalResult, // always a new fetch (previous entries removed above)
47
50
  ...githubResult,
51
+ ...bugzillaResult,
48
52
  ]));
49
53
 
50
54
  if (uniqueContributions.length === existingContributions.length) {
package/lib/github.js CHANGED
@@ -1,5 +1,6 @@
1
1
  const debug = require('debug')('contributions:github');
2
2
  const fetcher = require('./github-fetcher');
3
+ const { getPadded } = require('./utils');
3
4
 
4
5
  const ONE_YEAR_IN_MS = 1000 * 60 * 60 * 24 * 365;
5
6
  const DELAY_FETCHING_MS = 2000;
@@ -271,14 +272,6 @@ function getNextPeriod(stopDate, period, lowerBoundDate) {
271
272
  return [newBeforeDate, newAfterDate];
272
273
  }
273
274
 
274
- function getPadded(number) {
275
- if (number < 10) {
276
- return `0${number}`;
277
- }
278
-
279
- return `${number}`;
280
- }
281
-
282
275
  async function* periodGenerator(stopDate, lowerBoundDate) {
283
276
  const date = new Date();
284
277
  const firstOfThisMonth = `${date.getFullYear()}-${getPadded(date.getMonth() + 1)}-01`;
package/lib/utils.js ADDED
@@ -0,0 +1,11 @@
1
+ module.exports = {
2
+ getPadded,
3
+ };
4
+
5
+ function getPadded(number) {
6
+ if (number < 10) {
7
+ return `0${number}`;
8
+ }
9
+
10
+ return `${number}`;
11
+ }
@@ -0,0 +1,18 @@
1
+ const test = require('ava');
2
+
3
+ const utils = require('./utils');
4
+
5
+ test.serial('should pad number below 10', (t) => {
6
+ const padded = utils.getPadded(8);
7
+ t.is(padded, '08');
8
+ });
9
+
10
+ test.serial('should not pad 10', (t) => {
11
+ const padded = utils.getPadded(10);
12
+ t.is(padded, '10');
13
+ });
14
+
15
+ test.serial('should not pad 11', (t) => {
16
+ const padded = utils.getPadded(11);
17
+ t.is(padded, '11');
18
+ });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "volunteer-contributions-fetch",
3
- "version": "2.2.0",
3
+ "version": "2.3.0",
4
4
  "description": "Library to fetch volunteer contributions from several sources such as GitHub",
5
5
  "author": "Michael Kohler",
6
6
  "license": "MIT",