volunteer-contributions-fetch 2.3.7 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/test.yml +1 -1
- package/README.md +9 -7
- package/lib/contributions.js +23 -0
- package/lib/contributions.test.js +41 -0
- package/lib/discourse.js +21 -2
- package/lib/discourse.test.js +54 -0
- package/lib/fetch.js +3 -1
- package/package.json +14 -14
package/README.md
CHANGED
|
@@ -109,13 +109,14 @@ This section documents all the possible configuration values. For examples see t
|
|
|
109
109
|
|
|
110
110
|
#### Discourse
|
|
111
111
|
|
|
112
|
-
| Field
|
|
113
|
-
|
|
|
114
|
-
| `enabled`
|
|
115
|
-
| `baseUrl`
|
|
116
|
-
| `username`
|
|
117
|
-
| `topicType`
|
|
118
|
-
| `postType`
|
|
112
|
+
| Field | Data Type | Required | Default | Description |
|
|
113
|
+
| ----------------- | --------- | -------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
|
|
114
|
+
| `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `discourse` object entirely. |
|
|
115
|
+
| `baseUrl` | string | Yes | - | API base url without trailing slash |
|
|
116
|
+
| `username` | string | Yes | - | Username on Discourse |
|
|
117
|
+
| `topicType` | string | No | Created Discourse Topic | Label for entries related to having created a new topic. |
|
|
118
|
+
| `postType` | string | No | Posted on Discourse Topic | Label for entries related to having posted on an existing topic. |
|
|
119
|
+
| `keepDeletedPost` | boolean | No | false | If a post gets deleted and can't be queried anymore, setting this to `true` will not delete it from the previous contributions. |
|
|
119
120
|
|
|
120
121
|
#### GitHub
|
|
121
122
|
|
|
@@ -176,6 +177,7 @@ const config = {
|
|
|
176
177
|
username: 'mkohler',
|
|
177
178
|
topicType: 'Created Discourse Topic',
|
|
178
179
|
postType: 'Posted on Discourse Topic',
|
|
180
|
+
keepDeletedPost: true,
|
|
179
181
|
},
|
|
180
182
|
mediaWiki: {
|
|
181
183
|
enabled: true,
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
const ensureUniqueContributions = (allContributions) => {
|
|
2
|
+
const uniqueContributions = allContributions.reduce((acc, contribution) => {
|
|
3
|
+
const existingContribution = acc.find(
|
|
4
|
+
(uniqueContribution) =>
|
|
5
|
+
uniqueContribution.source === contribution.source &&
|
|
6
|
+
uniqueContribution.type === contribution.type &&
|
|
7
|
+
uniqueContribution.createdAt.getTime() ===
|
|
8
|
+
contribution.createdAt.getTime()
|
|
9
|
+
);
|
|
10
|
+
|
|
11
|
+
if (!existingContribution) {
|
|
12
|
+
acc.push(contribution);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
return acc;
|
|
16
|
+
}, []);
|
|
17
|
+
|
|
18
|
+
return uniqueContributions;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
module.exports = {
|
|
22
|
+
ensureUniqueContributions,
|
|
23
|
+
};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
const test = require('ava');
|
|
2
|
+
const { ensureUniqueContributions } = require('./contributions');
|
|
3
|
+
|
|
4
|
+
const contributions = [
|
|
5
|
+
{
|
|
6
|
+
createdAt: new Date('2022-12-08T22:43:17.000Z'),
|
|
7
|
+
description: 'fix: remove unapproved sv-SE Gutenberg sentences',
|
|
8
|
+
link: 'https://github.com/common-voice/sentence-collector/pull/656',
|
|
9
|
+
type: 'Created PR',
|
|
10
|
+
source: 'github-pr-created',
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
createdAt: new Date('2022-12-05T18:55:04.129Z'),
|
|
14
|
+
description:
|
|
15
|
+
'High amount of low quality submissions in Sentence Collector makes reviewing boring',
|
|
16
|
+
link: 'https://discourse.mozilla.org/t/high-amount-of-low-quality-submissions-in-sentence-collector-makes-reviewing-boring/108368/4',
|
|
17
|
+
type: 'Posted on Discourse Topic',
|
|
18
|
+
source: 'discourse-posts',
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
createdAt: new Date('2022-12-05T18:55:04.129Z'),
|
|
22
|
+
description:
|
|
23
|
+
'High amount of low quality submissions in Sentence Collector makes reviewing boring',
|
|
24
|
+
link: 'https://discourse.mozilla.org/t/high-amount-of-low-quality-submissions-in-sentence-collector-makes-reviewing-boring/108368/4',
|
|
25
|
+
type: 'Posted on Discourse Topic',
|
|
26
|
+
source: 'discourse-posts',
|
|
27
|
+
},
|
|
28
|
+
];
|
|
29
|
+
|
|
30
|
+
const alreadyUniqueContributions = [contributions[0], contributions[1]];
|
|
31
|
+
|
|
32
|
+
test('should not remove anything if there was nothing to do', async (t) => {
|
|
33
|
+
const result = ensureUniqueContributions(alreadyUniqueContributions);
|
|
34
|
+
t.deepEqual(result, alreadyUniqueContributions);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test('should ensure uniqueness', async (t) => {
|
|
38
|
+
const result = ensureUniqueContributions(contributions);
|
|
39
|
+
t.deepEqual(result, alreadyUniqueContributions);
|
|
40
|
+
t.is(result.length, 2);
|
|
41
|
+
});
|
package/lib/discourse.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const debug = require('debug')('contributions:discourse');
|
|
2
2
|
const axios = require('axios');
|
|
3
|
+
const { ensureUniqueContributions } = require('./contributions');
|
|
3
4
|
|
|
4
5
|
const RESULTS_PER_PAGE = 30;
|
|
5
6
|
const TOPICS_CATEGORY = 'discourse-topics';
|
|
@@ -31,7 +32,7 @@ function validate(config) {
|
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
34
|
|
|
34
|
-
async function gather({ discourse }) {
|
|
35
|
+
async function gather({ discourse }, existingContributions = []) {
|
|
35
36
|
if (!discourse || (discourse && !discourse.enabled)) {
|
|
36
37
|
debug('Discourse source not enabled, skipping');
|
|
37
38
|
return [];
|
|
@@ -39,7 +40,25 @@ async function gather({ discourse }) {
|
|
|
39
40
|
|
|
40
41
|
const topics = await processTopics(discourse);
|
|
41
42
|
const posts = await processPosts(discourse);
|
|
42
|
-
|
|
43
|
+
|
|
44
|
+
let contributions = [...topics, ...posts];
|
|
45
|
+
if (discourse.keepDeletedPost) {
|
|
46
|
+
debug('Keeping deleted posts and filtering by uniqueness');
|
|
47
|
+
const existingDiscourseContributions = existingContributions.filter(
|
|
48
|
+
(contribution) =>
|
|
49
|
+
contribution.source === TOPICS_CATEGORY ||
|
|
50
|
+
contribution.source === POSTS_CATEGORY
|
|
51
|
+
);
|
|
52
|
+
const allContributions = [
|
|
53
|
+
...existingDiscourseContributions.map((contribution) => ({
|
|
54
|
+
...contribution,
|
|
55
|
+
createdAt: new Date(contribution.createdAt),
|
|
56
|
+
})),
|
|
57
|
+
...topics,
|
|
58
|
+
...posts,
|
|
59
|
+
];
|
|
60
|
+
contributions = ensureUniqueContributions(allContributions);
|
|
61
|
+
}
|
|
43
62
|
|
|
44
63
|
debug('Finished gathering contributions');
|
|
45
64
|
return contributions;
|
package/lib/discourse.test.js
CHANGED
|
@@ -28,6 +28,12 @@ const TOPICS_SMALL = {
|
|
|
28
28
|
},
|
|
29
29
|
};
|
|
30
30
|
|
|
31
|
+
const TOPICS_EMPTY = {
|
|
32
|
+
topic_list: {
|
|
33
|
+
topics: [],
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
|
|
31
37
|
const POSTS = {
|
|
32
38
|
user_actions: new Array(30).fill(0).map((_, index) => ({
|
|
33
39
|
created_at: new Date('2020-04-13'),
|
|
@@ -173,3 +179,51 @@ test.serial('should format with custom types', async (t) => {
|
|
|
173
179
|
source: 'discourse-posts',
|
|
174
180
|
});
|
|
175
181
|
});
|
|
182
|
+
|
|
183
|
+
test.serial('should keep deleted post', async (t) => {
|
|
184
|
+
const specificTypeConfig = {
|
|
185
|
+
discourse: {
|
|
186
|
+
...config.discourse,
|
|
187
|
+
topicType: 'Topic created!',
|
|
188
|
+
postType: 'Posted!',
|
|
189
|
+
keepDeletedPost: true,
|
|
190
|
+
},
|
|
191
|
+
};
|
|
192
|
+
|
|
193
|
+
const existingContributions = [
|
|
194
|
+
{
|
|
195
|
+
createdAt: new Date('2020-04-13'),
|
|
196
|
+
description: 'Topic 0',
|
|
197
|
+
link: 'https://discourse.mozilla.org/t/slug-0/0',
|
|
198
|
+
type: 'Topic created!',
|
|
199
|
+
source: 'discourse-topics',
|
|
200
|
+
},
|
|
201
|
+
];
|
|
202
|
+
|
|
203
|
+
axios.get.onCall(0).resolves({ data: TOPICS_EMPTY });
|
|
204
|
+
axios.get.onCall(1).resolves({ data: POSTS_SMALL });
|
|
205
|
+
|
|
206
|
+
const contributions = await discourse.gather(
|
|
207
|
+
specificTypeConfig,
|
|
208
|
+
existingContributions
|
|
209
|
+
);
|
|
210
|
+
|
|
211
|
+
t.is(contributions.length, 2);
|
|
212
|
+
// Topic
|
|
213
|
+
// This topic was not returned from the response, but is an
|
|
214
|
+
// already existing contribution passed in.
|
|
215
|
+
t.deepEqual(contributions[0], {
|
|
216
|
+
createdAt: new Date('2020-04-13'),
|
|
217
|
+
description: 'Topic 0',
|
|
218
|
+
link: 'https://discourse.mozilla.org/t/slug-0/0',
|
|
219
|
+
type: specificTypeConfig.discourse.topicType,
|
|
220
|
+
source: 'discourse-topics',
|
|
221
|
+
});
|
|
222
|
+
t.deepEqual(contributions[1], {
|
|
223
|
+
createdAt: new Date('2020-04-13'),
|
|
224
|
+
description: 'Post 1',
|
|
225
|
+
link: 'https://discourse.mozilla.org/t/slug-1/1/1',
|
|
226
|
+
type: specificTypeConfig.discourse.postType,
|
|
227
|
+
source: 'discourse-posts',
|
|
228
|
+
});
|
|
229
|
+
});
|
package/lib/fetch.js
CHANGED
|
@@ -20,12 +20,14 @@ async function fetchAll(config) {
|
|
|
20
20
|
const githubResult = await github.gather(config, existingContributions);
|
|
21
21
|
const wikiResult = await wiki.gather(config);
|
|
22
22
|
const communityPortalResult = await communityPortal.gather(config);
|
|
23
|
-
const discourseResult = await discourse.gather(config);
|
|
23
|
+
const discourseResult = await discourse.gather(config, existingContributions);
|
|
24
24
|
|
|
25
25
|
// Some types are fetched on every update run, therefore we want to filter them out for now.
|
|
26
26
|
// Otherwise we might end up with duplicate entries we'd need to filter.
|
|
27
27
|
// We need to make sure we always will save the "reps" source entries, as we do not fetch those
|
|
28
28
|
// anymore.
|
|
29
|
+
// This also includes sources that handle their own "existing contributions" keeping
|
|
30
|
+
// even if only new contributions are fetched.
|
|
29
31
|
const ALWAYS_FULL_FETCH_SOURCE_ENTRIES = [
|
|
30
32
|
communityPortal.EVENTS_CATEGORY,
|
|
31
33
|
communityPortal.CAMPAIGNS_CATEGORY,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "volunteer-contributions-fetch",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "Library to fetch volunteer contributions from several sources such as GitHub",
|
|
5
5
|
"author": "Michael Kohler",
|
|
6
6
|
"license": "MIT",
|
|
@@ -10,25 +10,25 @@
|
|
|
10
10
|
},
|
|
11
11
|
"main": "index.js",
|
|
12
12
|
"dependencies": {
|
|
13
|
-
"@octokit/rest": "^
|
|
14
|
-
"axios": "^1.
|
|
13
|
+
"@octokit/rest": "^20.0.1",
|
|
14
|
+
"axios": "^1.4.0",
|
|
15
15
|
"debug": "^4.3.4",
|
|
16
|
-
"jsdom": "^
|
|
17
|
-
"xml2js": "^0.
|
|
16
|
+
"jsdom": "^22.1.0",
|
|
17
|
+
"xml2js": "^0.6.2"
|
|
18
18
|
},
|
|
19
19
|
"devDependencies": {
|
|
20
|
-
"@commitlint/cli": "^17.
|
|
21
|
-
"@commitlint/config-angular": "^17.
|
|
22
|
-
"ava": "^5.
|
|
23
|
-
"eslint": "^8.
|
|
20
|
+
"@commitlint/cli": "^17.7.1",
|
|
21
|
+
"@commitlint/config-angular": "^17.7.0",
|
|
22
|
+
"ava": "^5.3.1",
|
|
23
|
+
"eslint": "^8.47.0",
|
|
24
24
|
"eslint-config-airbnb-base": "^15.0.0",
|
|
25
|
-
"eslint-config-prettier": "^
|
|
26
|
-
"eslint-plugin-prettier": "^
|
|
25
|
+
"eslint-config-prettier": "^9.0.0",
|
|
26
|
+
"eslint-plugin-prettier": "^5.0.0",
|
|
27
27
|
"husky": "^8.0.3",
|
|
28
28
|
"nyc": "^15.1.0",
|
|
29
|
-
"prettier": "^
|
|
30
|
-
"semantic-release": "^21.0.
|
|
31
|
-
"sinon": "^15.0
|
|
29
|
+
"prettier": "^3.0.2",
|
|
30
|
+
"semantic-release": "^21.0.7",
|
|
31
|
+
"sinon": "^15.2.0"
|
|
32
32
|
},
|
|
33
33
|
"scripts": {
|
|
34
34
|
"prepare": "husky install",
|