volunteer-contributions-fetch 2.3.7 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,7 +12,7 @@ jobs:
12
12
  - name: Use Node.js 16
13
13
  uses: actions/setup-node@v3
14
14
  with:
15
- node-version: 16
15
+ node-version: 18
16
16
  - name: Install Dependencies
17
17
  run: npm ci
18
18
  - name: Linting
package/README.md CHANGED
@@ -109,13 +109,14 @@ This section documents all the possible configuration values. For examples see t
109
109
 
110
110
  #### Discourse
111
111
 
112
- | Field | Data Type | Required | Default | Description |
113
- | ----------- | --------- | -------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------- |
114
- | `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `discourse` object entirely. |
115
- | `baseUrl` | string | Yes | - | API base url without trailing slash |
116
- | `username` | string | Yes | - | Username on Discourse |
117
- | `topicType` | string | No | Created Discourse Topic | Label for entries related to having created a new topic. |
118
- | `postType` | string | No | Posted on Discourse Topic | Label for entries related to having posted on an existing topic. |
112
+ | Field | Data Type | Required | Default | Description |
113
+ | ----------------- | --------- | -------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
114
+ | `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `discourse` object entirely. |
115
+ | `baseUrl` | string | Yes | - | API base url without trailing slash |
116
+ | `username` | string | Yes | - | Username on Discourse |
117
+ | `topicType` | string | No | Created Discourse Topic | Label for entries related to having created a new topic. |
118
+ | `postType` | string | No | Posted on Discourse Topic | Label for entries related to having posted on an existing topic. |
119
+ | `keepDeletedPost` | boolean | No | false | If a post gets deleted and can't be queried anymore, setting this to `true` will not delete it from the previous contributions. |
119
120
 
120
121
  #### GitHub
121
122
 
@@ -176,6 +177,7 @@ const config = {
176
177
  username: 'mkohler',
177
178
  topicType: 'Created Discourse Topic',
178
179
  postType: 'Posted on Discourse Topic',
180
+ keepDeletedPost: true,
179
181
  },
180
182
  mediaWiki: {
181
183
  enabled: true,
@@ -0,0 +1,23 @@
1
+ const ensureUniqueContributions = (allContributions) => {
2
+ const uniqueContributions = allContributions.reduce((acc, contribution) => {
3
+ const existingContribution = acc.find(
4
+ (uniqueContribution) =>
5
+ uniqueContribution.source === contribution.source &&
6
+ uniqueContribution.type === contribution.type &&
7
+ uniqueContribution.createdAt.getTime() ===
8
+ contribution.createdAt.getTime()
9
+ );
10
+
11
+ if (!existingContribution) {
12
+ acc.push(contribution);
13
+ }
14
+
15
+ return acc;
16
+ }, []);
17
+
18
+ return uniqueContributions;
19
+ };
20
+
21
+ module.exports = {
22
+ ensureUniqueContributions,
23
+ };
@@ -0,0 +1,41 @@
1
+ const test = require('ava');
2
+ const { ensureUniqueContributions } = require('./contributions');
3
+
4
+ const contributions = [
5
+ {
6
+ createdAt: new Date('2022-12-08T22:43:17.000Z'),
7
+ description: 'fix: remove unapproved sv-SE Gutenberg sentences',
8
+ link: 'https://github.com/common-voice/sentence-collector/pull/656',
9
+ type: 'Created PR',
10
+ source: 'github-pr-created',
11
+ },
12
+ {
13
+ createdAt: new Date('2022-12-05T18:55:04.129Z'),
14
+ description:
15
+ 'High amount of low quality submissions in Sentence Collector makes reviewing boring',
16
+ link: 'https://discourse.mozilla.org/t/high-amount-of-low-quality-submissions-in-sentence-collector-makes-reviewing-boring/108368/4',
17
+ type: 'Posted on Discourse Topic',
18
+ source: 'discourse-posts',
19
+ },
20
+ {
21
+ createdAt: new Date('2022-12-05T18:55:04.129Z'),
22
+ description:
23
+ 'High amount of low quality submissions in Sentence Collector makes reviewing boring',
24
+ link: 'https://discourse.mozilla.org/t/high-amount-of-low-quality-submissions-in-sentence-collector-makes-reviewing-boring/108368/4',
25
+ type: 'Posted on Discourse Topic',
26
+ source: 'discourse-posts',
27
+ },
28
+ ];
29
+
30
+ const alreadyUniqueContributions = [contributions[0], contributions[1]];
31
+
32
+ test('should not remove anything if there was nothing to do', async (t) => {
33
+ const result = ensureUniqueContributions(alreadyUniqueContributions);
34
+ t.deepEqual(result, alreadyUniqueContributions);
35
+ });
36
+
37
+ test('should ensure uniqueness', async (t) => {
38
+ const result = ensureUniqueContributions(contributions);
39
+ t.deepEqual(result, alreadyUniqueContributions);
40
+ t.is(result.length, 2);
41
+ });
package/lib/discourse.js CHANGED
@@ -1,5 +1,6 @@
1
1
  const debug = require('debug')('contributions:discourse');
2
2
  const axios = require('axios');
3
+ const { ensureUniqueContributions } = require('./contributions');
3
4
 
4
5
  const RESULTS_PER_PAGE = 30;
5
6
  const TOPICS_CATEGORY = 'discourse-topics';
@@ -31,7 +32,7 @@ function validate(config) {
31
32
  }
32
33
  }
33
34
 
34
- async function gather({ discourse }) {
35
+ async function gather({ discourse }, existingContributions = []) {
35
36
  if (!discourse || (discourse && !discourse.enabled)) {
36
37
  debug('Discourse source not enabled, skipping');
37
38
  return [];
@@ -39,7 +40,25 @@ async function gather({ discourse }) {
39
40
 
40
41
  const topics = await processTopics(discourse);
41
42
  const posts = await processPosts(discourse);
42
- const contributions = [...topics, ...posts];
43
+
44
+ let contributions = [...topics, ...posts];
45
+ if (discourse.keepDeletedPost) {
46
+ debug('Keeping deleted posts and filtering by uniqueness');
47
+ const existingDiscourseContributions = existingContributions.filter(
48
+ (contribution) =>
49
+ contribution.source === TOPICS_CATEGORY ||
50
+ contribution.source === POSTS_CATEGORY
51
+ );
52
+ const allContributions = [
53
+ ...existingDiscourseContributions.map((contribution) => ({
54
+ ...contribution,
55
+ createdAt: new Date(contribution.createdAt),
56
+ })),
57
+ ...topics,
58
+ ...posts,
59
+ ];
60
+ contributions = ensureUniqueContributions(allContributions);
61
+ }
43
62
 
44
63
  debug('Finished gathering contributions');
45
64
  return contributions;
@@ -28,6 +28,12 @@ const TOPICS_SMALL = {
28
28
  },
29
29
  };
30
30
 
31
+ const TOPICS_EMPTY = {
32
+ topic_list: {
33
+ topics: [],
34
+ },
35
+ };
36
+
31
37
  const POSTS = {
32
38
  user_actions: new Array(30).fill(0).map((_, index) => ({
33
39
  created_at: new Date('2020-04-13'),
@@ -173,3 +179,51 @@ test.serial('should format with custom types', async (t) => {
173
179
  source: 'discourse-posts',
174
180
  });
175
181
  });
182
+
183
+ test.serial('should keep deleted post', async (t) => {
184
+ const specificTypeConfig = {
185
+ discourse: {
186
+ ...config.discourse,
187
+ topicType: 'Topic created!',
188
+ postType: 'Posted!',
189
+ keepDeletedPost: true,
190
+ },
191
+ };
192
+
193
+ const existingContributions = [
194
+ {
195
+ createdAt: new Date('2020-04-13'),
196
+ description: 'Topic 0',
197
+ link: 'https://discourse.mozilla.org/t/slug-0/0',
198
+ type: 'Topic created!',
199
+ source: 'discourse-topics',
200
+ },
201
+ ];
202
+
203
+ axios.get.onCall(0).resolves({ data: TOPICS_EMPTY });
204
+ axios.get.onCall(1).resolves({ data: POSTS_SMALL });
205
+
206
+ const contributions = await discourse.gather(
207
+ specificTypeConfig,
208
+ existingContributions
209
+ );
210
+
211
+ t.is(contributions.length, 2);
212
+ // Topic
213
+ // This topic was not returned from the response, but is an
214
+ // already existing contribution passed in.
215
+ t.deepEqual(contributions[0], {
216
+ createdAt: new Date('2020-04-13'),
217
+ description: 'Topic 0',
218
+ link: 'https://discourse.mozilla.org/t/slug-0/0',
219
+ type: specificTypeConfig.discourse.topicType,
220
+ source: 'discourse-topics',
221
+ });
222
+ t.deepEqual(contributions[1], {
223
+ createdAt: new Date('2020-04-13'),
224
+ description: 'Post 1',
225
+ link: 'https://discourse.mozilla.org/t/slug-1/1/1',
226
+ type: specificTypeConfig.discourse.postType,
227
+ source: 'discourse-posts',
228
+ });
229
+ });
package/lib/fetch.js CHANGED
@@ -20,12 +20,14 @@ async function fetchAll(config) {
20
20
  const githubResult = await github.gather(config, existingContributions);
21
21
  const wikiResult = await wiki.gather(config);
22
22
  const communityPortalResult = await communityPortal.gather(config);
23
- const discourseResult = await discourse.gather(config);
23
+ const discourseResult = await discourse.gather(config, existingContributions);
24
24
 
25
25
  // Some types are fetched on every update run, therefore we want to filter them out for now.
26
26
  // Otherwise we might end up with duplicate entries we'd need to filter.
27
27
  // We need to make sure we always will save the "reps" source entries, as we do not fetch those
28
28
  // anymore.
29
+ // This also includes sources that handle their own "existing contributions" keeping
30
+ // even if only new contributions are fetched.
29
31
  const ALWAYS_FULL_FETCH_SOURCE_ENTRIES = [
30
32
  communityPortal.EVENTS_CATEGORY,
31
33
  communityPortal.CAMPAIGNS_CATEGORY,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "volunteer-contributions-fetch",
3
- "version": "2.3.7",
3
+ "version": "2.4.0",
4
4
  "description": "Library to fetch volunteer contributions from several sources such as GitHub",
5
5
  "author": "Michael Kohler",
6
6
  "license": "MIT",
@@ -10,25 +10,25 @@
10
10
  },
11
11
  "main": "index.js",
12
12
  "dependencies": {
13
- "@octokit/rest": "^19.0.7",
14
- "axios": "^1.3.5",
13
+ "@octokit/rest": "^20.0.1",
14
+ "axios": "^1.4.0",
15
15
  "debug": "^4.3.4",
16
- "jsdom": "^21.1.1",
17
- "xml2js": "^0.5.0"
16
+ "jsdom": "^22.1.0",
17
+ "xml2js": "^0.6.2"
18
18
  },
19
19
  "devDependencies": {
20
- "@commitlint/cli": "^17.5.1",
21
- "@commitlint/config-angular": "^17.4.4",
22
- "ava": "^5.2.0",
23
- "eslint": "^8.38.0",
20
+ "@commitlint/cli": "^17.7.1",
21
+ "@commitlint/config-angular": "^17.7.0",
22
+ "ava": "^5.3.1",
23
+ "eslint": "^8.47.0",
24
24
  "eslint-config-airbnb-base": "^15.0.0",
25
- "eslint-config-prettier": "^8.8.0",
26
- "eslint-plugin-prettier": "^4.2.1",
25
+ "eslint-config-prettier": "^9.0.0",
26
+ "eslint-plugin-prettier": "^5.0.0",
27
27
  "husky": "^8.0.3",
28
28
  "nyc": "^15.1.0",
29
- "prettier": "^2.8.7",
30
- "semantic-release": "^21.0.1",
31
- "sinon": "^15.0.3"
29
+ "prettier": "^3.0.2",
30
+ "semantic-release": "^21.0.7",
31
+ "sinon": "^15.2.0"
32
32
  },
33
33
  "scripts": {
34
34
  "prepare": "husky install",