volunteer-contributions-fetch 3.1.2 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -24
- package/lib/bugzilla.js +73 -32
- package/lib/community-portal.js +96 -25
- package/lib/contributions.js +74 -12
- package/lib/discourse.js +50 -22
- package/lib/fetch.js +85 -39
- package/lib/github-fetcher.js +10 -9
- package/lib/github.js +353 -272
- package/lib/osm.js +61 -21
- package/lib/utils.js +52 -0
- package/lib/validator.js +2 -2
- package/lib/wiki.js +66 -50
- package/package.json +21 -33
- package/.editorconfig +0 -13
- package/.env.example +0 -1
- package/.eslintrc +0 -23
- package/.github/workflows/release.yml +0 -31
- package/.github/workflows/test.yml +0 -20
- package/.husky/commit-msg +0 -1
- package/.husky/pre-commit +0 -2
- package/.prettierignore +0 -4
- package/commitlint.config.cjs +0 -6
- package/example/index.js +0 -36
- package/fixtures/community-portal.js +0 -265
- package/fixtures/osm.js +0 -28
- package/fixtures/wiki.js +0 -52
- package/lib/bugzilla.test.js +0 -204
- package/lib/community-portal.test.js +0 -154
- package/lib/contributions.test.js +0 -42
- package/lib/discourse.test.js +0 -221
- package/lib/fetch.test.js +0 -114
- package/lib/github.test.js +0 -1355
- package/lib/osm.test.js +0 -102
- package/lib/utils.test.js +0 -18
- package/lib/validator.test.js +0 -50
- package/lib/wiki.test.js +0 -126
package/README.md
CHANGED
|
@@ -9,6 +9,7 @@ This library fetches contributions from several different sources and saves them
|
|
|
9
9
|
Currently the following sources are supported. Feel free to contribute a PR if you'd like to support another source!
|
|
10
10
|
|
|
11
11
|
- GitHub
|
|
12
|
+
- Commits
|
|
12
13
|
- Issues created
|
|
13
14
|
- Comments on issues
|
|
14
15
|
- PRs
|
|
@@ -45,7 +46,7 @@ The following sources would be interesting for (at least) Mozilla contributors,
|
|
|
45
46
|
First install the package from npm:
|
|
46
47
|
|
|
47
48
|
```bash
|
|
48
|
-
npm install volunteer-
|
|
49
|
+
npm install volunteer-contributions-fetch
|
|
49
50
|
```
|
|
50
51
|
|
|
51
52
|
Then you can create a file in your project which runs the fetch operation and provides the necessary config:
|
|
@@ -68,7 +69,7 @@ import { fetchAll } from 'volunteer-contributions-fetch';
|
|
|
68
69
|
const config = {}; // See below for documentation and a sample config
|
|
69
70
|
const existingContributions = [{ ... }];
|
|
70
71
|
|
|
71
|
-
const results = fetchAll(config, existingContributions);
|
|
72
|
+
const results = await fetchAll(config, existingContributions);
|
|
72
73
|
// results is now all existing contributions plus any newly fetched contribution
|
|
73
74
|
```
|
|
74
75
|
|
|
@@ -76,7 +77,7 @@ Note that passing existing contributions this way will be ignored if a `outputFi
|
|
|
76
77
|
|
|
77
78
|
## Configuration
|
|
78
79
|
|
|
79
|
-
You can configure the different providers with a config you pass into the `fetchAll` function call. These config parameters are outlined below. The config gets validated and you might get an error if you haven't specified all necessary properties.
|
|
80
|
+
You can configure the different providers with a config you pass into the `fetchAll` function call. These config parameters are outlined below. The config gets validated and you might get an error if you haven't specified all necessary properties or if the config contains unknown top-level keys (for example a misspelled source name).
|
|
80
81
|
|
|
81
82
|
Additionally to the configuration file, we also require some environment variables to be set:
|
|
82
83
|
|
|
@@ -98,18 +99,21 @@ This section documents all the possible configuration values. For examples see t
|
|
|
98
99
|
| `discourse` | Object | No | - | Discourse configuration options (see below) |
|
|
99
100
|
| `github` | Object | No | - | GitHub configuration options (see below) |
|
|
100
101
|
| `mediaWiki` | Object | No | - | MediaWiki configuration options (see below) |
|
|
102
|
+
| `osm` | Object | No | - | OpenStreetMap configuration options (see below) |
|
|
103
|
+
|
|
104
|
+
The fields marked as required below are only required if the source is enabled. To disable a source, `{ enabled: false }` is enough and any other fields are ignored.
|
|
101
105
|
|
|
102
106
|
#### Bugzilla
|
|
103
107
|
|
|
104
|
-
| Field | Data Type | Required | Default | Description
|
|
105
|
-
| --------------------- | --------- | -------- | ------------------------- |
|
|
106
|
-
| `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `
|
|
107
|
-
| `baseUrl` | string | Yes | - | API base url without trailing slash (for example: `https://bugzilla.mozilla.org`)
|
|
108
|
-
| `username` | string | Yes | - | Username on
|
|
109
|
-
| `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track
|
|
110
|
-
| `types` | Object | No | - | Object containing the different labels below
|
|
111
|
-
| `types.createdType` | string | No | Created a Bug Report | Label for entries related to having report a bug/enhancement.
|
|
112
|
-
| `types.commentedType` | string | No | Commented on a Bug Report | Label for entries related to having commented on a bug/enhancement.
|
|
108
|
+
| Field | Data Type | Required | Default | Description |
|
|
109
|
+
| --------------------- | --------- | -------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
|
110
|
+
| `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `bugzilla` object entirely. |
|
|
111
|
+
| `baseUrl` | string | Yes | - | API base url without trailing slash (for example: `https://bugzilla.mozilla.org`) |
|
|
112
|
+
| `username` | string | Yes | - | Username on Bugzilla |
|
|
113
|
+
| `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track |
|
|
114
|
+
| `types` | Object | No | - | Object containing the different labels below |
|
|
115
|
+
| `types.createdType` | string | No | Created a Bug Report | Label for entries related to having report a bug/enhancement. |
|
|
116
|
+
| `types.commentedType` | string | No | Commented on a Bug Report | Label for entries related to having commented on a bug/enhancement. |
|
|
113
117
|
|
|
114
118
|
#### Community Portal
|
|
115
119
|
|
|
@@ -144,14 +148,14 @@ This section documents all the possible configuration values. For examples see t
|
|
|
144
148
|
| `reviewsEnabled` | boolean | No | true | Indicates whether fetching reviews is enabled. |
|
|
145
149
|
| `username` | string | Yes | - | Username on GitHub |
|
|
146
150
|
| `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track |
|
|
147
|
-
| `filter` | string | Yes | - | Regex to apply to each entry to filter for organization/repo names. For example:
|
|
151
|
+
| `filter` | string | Yes | - | Regex to apply to each entry to filter for organization/repo names. For example: `mozilla` to only include orgs and repos that contain "mozilla". |
|
|
148
152
|
| `types` | Object | No | - | Object containing the different labels below |
|
|
149
153
|
| `types.commit` | string | No | GitHub Commit | Label for entries related to GitHub commits |
|
|
150
154
|
| `types.privateCommit` | string | No | Commit in private repository | Label for entries related to private GitHub commits |
|
|
151
155
|
| `types.createdPR` | string | No | Created PR | Label for entries related to opening GitHub PRs |
|
|
152
156
|
| `types.createdIssue` | string | No | Created Issue Report | Label for entries related to creating GitHub issues |
|
|
153
157
|
| `types.commented` | string | No | Commented on an Issue | Label for entries related to commenting on an issue |
|
|
154
|
-
| `types.commentedPR` | string | No | Commented on a Pull Request | Label for entries related to commenting on a GitHub PR (review)
|
|
158
|
+
| `types.commentedPR` | string | No | Commented on a Pull Request | Label for entries related to commenting on a GitHub PR (comment or review) |
|
|
155
159
|
| `types.approvedPR` | string | No | Approved a Pull Request | Label for entries related to approving a GitHub PR |
|
|
156
160
|
| `types.changesRequestedPR` | string | No | Requested changes on a Pull Request | Label for entries related to requesting changes on a GitHub PR |
|
|
157
161
|
| `types.reviewedPR` | string | No | Reviewed a Pull Request | Label for entries related to reviewing a GitHub PR (fallback) |
|
|
@@ -159,13 +163,13 @@ This section documents all the possible configuration values. For examples see t
|
|
|
159
163
|
|
|
160
164
|
#### MediaWiki
|
|
161
165
|
|
|
162
|
-
| Field | Data Type | Required | Default | Description
|
|
163
|
-
| ---------- | --------- | -------- | --------- |
|
|
164
|
-
| `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `mediaWiki` object entirely.
|
|
165
|
-
| `baseUrl` | string | Yes | - | API
|
|
166
|
-
| `username` | string | Yes | - | Username on MediaWiki
|
|
167
|
-
| `editType` | string | No | Wiki Edit | Label for entries related to having edited an article.
|
|
168
|
-
| `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track
|
|
166
|
+
| Field | Data Type | Required | Default | Description |
|
|
167
|
+
| ---------- | --------- | -------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
168
|
+
| `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `mediaWiki` object entirely. |
|
|
169
|
+
| `baseUrl` | string | Yes | - | `feedcontributions` API URL including its query string, e.g. `https://wiki.mozilla.org/api.php?action=feedcontributions`. `&user=`, `&year=` and `&month=` are appended to it. |
|
|
170
|
+
| `username` | string | Yes | - | Username on MediaWiki |
|
|
171
|
+
| `editType` | string | No | Wiki Edit | Label for entries related to having edited an article. |
|
|
172
|
+
| `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track |
|
|
169
173
|
|
|
170
174
|
#### OpenStreetMaps
|
|
171
175
|
|
|
@@ -179,7 +183,7 @@ This section documents all the possible configuration values. For examples see t
|
|
|
179
183
|
|
|
180
184
|
```js
|
|
181
185
|
const config = {
|
|
182
|
-
outputFile: `${
|
|
186
|
+
outputFile: `${import.meta.dirname}/_data/contributions.json`,
|
|
183
187
|
bugzilla: {
|
|
184
188
|
enabled: true,
|
|
185
189
|
baseUrl: 'https://bugzilla.mozilla.org',
|
|
@@ -264,8 +268,6 @@ You then can use `npm run example` which uses `fetchAll` if you would like to te
|
|
|
264
268
|
npm run eslint
|
|
265
269
|
```
|
|
266
270
|
|
|
267
|
-
Note that due to the AirBnB config we are still using ESLint 8.
|
|
268
|
-
|
|
269
271
|
### Deployments
|
|
270
272
|
|
|
271
273
|
Please use [Angular Commit Message Conventions](https://github.com/angular/angular/blob/master/CONTRIBUTING.md#-commit-message-format) when contributing to this project, as depending on the commit message the GitHub Action will directly publish a new version of the package.
|
package/lib/bugzilla.js
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
import Debug from 'debug';
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
baseUrlSchema,
|
|
6
|
+
FETCH_TIMEOUT,
|
|
7
|
+
findLatest,
|
|
8
|
+
getPadded,
|
|
9
|
+
sourceSchema,
|
|
10
|
+
stopDateSchema,
|
|
11
|
+
} from './utils.js';
|
|
5
12
|
|
|
6
13
|
const debug = Debug('contributions:bugzilla');
|
|
7
14
|
|
|
@@ -12,28 +19,32 @@ const TYPE_CREATED = 'Created a Bug Report';
|
|
|
12
19
|
const TYPE_COMMENTED = 'Commented on a Bug Report';
|
|
13
20
|
|
|
14
21
|
const RESULTS_PER_PAGE = 500;
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
.
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
22
|
+
|
|
23
|
+
const BUG_DOES_NOT_EXIST = 101;
|
|
24
|
+
const BUG_ACCESS_DENIED = 102;
|
|
25
|
+
|
|
26
|
+
export const bugzillaSchema = sourceSchema(
|
|
27
|
+
z.strictObject({
|
|
28
|
+
enabled: z.literal(true),
|
|
29
|
+
baseUrl: baseUrlSchema,
|
|
30
|
+
username: z.string().min(1),
|
|
31
|
+
stopDate: stopDateSchema,
|
|
32
|
+
types: z
|
|
33
|
+
.strictObject({
|
|
34
|
+
createdType: z.string().optional(),
|
|
35
|
+
commentedType: z.string().optional(),
|
|
36
|
+
})
|
|
37
|
+
.optional(),
|
|
38
|
+
})
|
|
39
|
+
);
|
|
29
40
|
|
|
30
41
|
export async function gather({ bugzilla }, existingContributions = []) {
|
|
31
|
-
if (!bugzilla
|
|
42
|
+
if (!bugzilla?.enabled) {
|
|
32
43
|
debug('Bugzilla source not enabled, skipping');
|
|
33
44
|
return [];
|
|
34
45
|
}
|
|
35
46
|
|
|
36
|
-
const latest = existingContributions
|
|
47
|
+
const latest = findLatest(existingContributions, (contribution) => {
|
|
37
48
|
return (
|
|
38
49
|
contribution.source === BUGZILLA_COMMENTED_CATEGORY ||
|
|
39
50
|
contribution.source === BUGZILLA_CREATED_CATEGORY
|
|
@@ -71,26 +82,29 @@ export async function gather({ bugzilla }, existingContributions = []) {
|
|
|
71
82
|
}
|
|
72
83
|
|
|
73
84
|
async function* bugPageGenerator(bugzilla, stopDate) {
|
|
74
|
-
let hasMoreToFetch = true;
|
|
75
85
|
let offset = 0;
|
|
76
86
|
const date = new Date(stopDate);
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
)}-${getPadded(
|
|
80
|
-
|
|
81
|
-
|
|
87
|
+
// Dates are parsed as UTC midnight, so we need to use the UTC methods,
|
|
88
|
+
// otherwise time zones behind UTC end up in the previous day.
|
|
89
|
+
const stop = `${date.getUTCFullYear()}-${getPadded(
|
|
90
|
+
date.getUTCMonth() + 1
|
|
91
|
+
)}-${getPadded(date.getUTCDate())}`;
|
|
92
|
+
while (true) {
|
|
93
|
+
const query = `quicksearch=ALL%20commenter:${encodeURIComponent(bugzilla.username)}&limit=${RESULTS_PER_PAGE}&offset=${offset}&order=bug_id&last_change_time=${stop}`;
|
|
82
94
|
const bugsUrl = `${bugzilla.baseUrl}/rest/bug?${query}`;
|
|
83
95
|
debug(`Fetching from ${bugsUrl}`);
|
|
84
|
-
// eslint-disable-next-line no-await-in-loop
|
|
85
96
|
const response = await fetch(bugsUrl, {
|
|
86
97
|
signal: AbortSignal.timeout(FETCH_TIMEOUT),
|
|
87
98
|
});
|
|
88
|
-
|
|
99
|
+
if (!response.ok) {
|
|
100
|
+
throw new Error(
|
|
101
|
+
`Bugzilla request to ${bugsUrl} failed with status ${response.status}`
|
|
102
|
+
);
|
|
103
|
+
}
|
|
89
104
|
const data = await response.json();
|
|
90
105
|
debug(`Got ${data.bugs.length} bugs results`);
|
|
91
106
|
// If we have less results than the max result, we're at the end
|
|
92
107
|
if (data.bugs.length < RESULTS_PER_PAGE) {
|
|
93
|
-
hasMoreToFetch = false;
|
|
94
108
|
yield data.bugs;
|
|
95
109
|
return;
|
|
96
110
|
}
|
|
@@ -101,17 +115,29 @@ async function* bugPageGenerator(bugzilla, stopDate) {
|
|
|
101
115
|
}
|
|
102
116
|
|
|
103
117
|
async function* getComments(bugzilla, bugs, titleMap) {
|
|
118
|
+
const username = bugzilla.username.toLowerCase();
|
|
104
119
|
for await (const bug of bugs) {
|
|
105
120
|
const commentsUrl = `${bugzilla.baseUrl}/rest/bug/${bug.id}/comment`;
|
|
106
121
|
debug(`Fetching from ${commentsUrl}`);
|
|
107
|
-
// eslint-disable-next-line no-await-in-loop
|
|
108
122
|
const response = await fetch(commentsUrl, {
|
|
109
123
|
signal: AbortSignal.timeout(FETCH_TIMEOUT),
|
|
110
124
|
});
|
|
125
|
+
// A deleted or restricted bug has nothing left to fetch, so we skip it.
|
|
126
|
+
// Any other error must abort the run, as otherwise the remaining bugs
|
|
127
|
+
// would be skipped and the next run would start past them.
|
|
128
|
+
if (await isGoneResponse(response)) {
|
|
129
|
+
debug(`Skipping unavailable bug ${bug.id}`);
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
if (!response.ok) {
|
|
133
|
+
throw new Error(
|
|
134
|
+
`Bugzilla request to ${commentsUrl} failed with status ${response.status}`
|
|
135
|
+
);
|
|
136
|
+
}
|
|
111
137
|
const data = await response.json();
|
|
112
138
|
|
|
113
139
|
const ownComments = data.bugs[bug.id].comments.filter(
|
|
114
|
-
(comment) => comment.creator ===
|
|
140
|
+
(comment) => comment.creator?.toLowerCase() === username
|
|
115
141
|
);
|
|
116
142
|
|
|
117
143
|
yield processEntities(bugzilla, ownComments, titleMap);
|
|
@@ -120,18 +146,33 @@ async function* getComments(bugzilla, bugs, titleMap) {
|
|
|
120
146
|
return [];
|
|
121
147
|
}
|
|
122
148
|
|
|
123
|
-
function
|
|
124
|
-
|
|
149
|
+
async function isGoneResponse(response) {
|
|
150
|
+
if (![401, 403, 404].includes(response.status)) {
|
|
151
|
+
return false;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
try {
|
|
155
|
+
const data = await response.json();
|
|
156
|
+
return [BUG_DOES_NOT_EXIST, BUG_ACCESS_DENIED].includes(data?.code);
|
|
157
|
+
} catch {
|
|
158
|
+
return false;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function processEntities({ baseUrl, types }, entities, titleMap) {
|
|
163
|
+
return entities.map((entity) =>
|
|
164
|
+
format(baseUrl, types || {}, entity, titleMap)
|
|
165
|
+
);
|
|
125
166
|
}
|
|
126
167
|
|
|
127
|
-
function format(types, entity, titleMap) {
|
|
168
|
+
function format(baseUrl, types, entity, titleMap) {
|
|
128
169
|
const created = types.createdType || TYPE_CREATED;
|
|
129
170
|
const commented = types.commentedType || TYPE_COMMENTED;
|
|
130
171
|
|
|
131
172
|
return {
|
|
132
173
|
createdAt: new Date(entity.creation_time),
|
|
133
174
|
description: titleMap.get(entity.bug_id),
|
|
134
|
-
link:
|
|
175
|
+
link: `${baseUrl}/show_bug.cgi?id=${entity.bug_id}#c${entity.count}`,
|
|
135
176
|
type: entity.count === 0 ? created : commented,
|
|
136
177
|
source:
|
|
137
178
|
entity.count === 0
|
package/lib/community-portal.js
CHANGED
|
@@ -2,6 +2,9 @@ import Debug from 'debug';
|
|
|
2
2
|
import { JSDOM } from 'jsdom';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
|
|
5
|
+
import { ensureUniqueContributions } from './contributions.js';
|
|
6
|
+
import { baseUrlSchema, FETCH_TIMEOUT, sourceSchema } from './utils.js';
|
|
7
|
+
|
|
5
8
|
const debug = Debug('contributions:community-portal');
|
|
6
9
|
|
|
7
10
|
export const EVENTS_CATEGORY = 'community-portal-events';
|
|
@@ -10,43 +13,59 @@ export const CAMPAIGNS_CATEGORY = 'community-portal-campaigns';
|
|
|
10
13
|
const TYPE_PARTICIPATION = 'Participated in an event';
|
|
11
14
|
const TYPE_CAMPAIGN = 'Participated in a campaign';
|
|
12
15
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
16
|
+
const MONTHS = [
|
|
17
|
+
'january',
|
|
18
|
+
'february',
|
|
19
|
+
'march',
|
|
20
|
+
'april',
|
|
21
|
+
'may',
|
|
22
|
+
'june',
|
|
23
|
+
'july',
|
|
24
|
+
'august',
|
|
25
|
+
'september',
|
|
26
|
+
'october',
|
|
27
|
+
'november',
|
|
28
|
+
'december',
|
|
29
|
+
];
|
|
30
|
+
|
|
31
|
+
export const communityPortalSchema = sourceSchema(
|
|
32
|
+
z.strictObject({
|
|
33
|
+
enabled: z.literal(true),
|
|
34
|
+
baseUrl: baseUrlSchema,
|
|
35
|
+
username: z.string().min(1),
|
|
36
|
+
participationType: z.string().optional(),
|
|
37
|
+
campaignType: z.string().optional(),
|
|
38
|
+
})
|
|
39
|
+
);
|
|
20
40
|
|
|
21
41
|
export async function gather({ communityPortal }) {
|
|
22
|
-
if (!communityPortal
|
|
42
|
+
if (!communityPortal?.enabled) {
|
|
23
43
|
debug('Community Portal source not enabled, skipping');
|
|
24
44
|
return [];
|
|
25
45
|
}
|
|
26
46
|
|
|
27
47
|
const profileUrl = `${communityPortal.baseUrl}/${communityPortal.username}`;
|
|
28
48
|
const document = await getCommunityPortalDocument(profileUrl);
|
|
29
|
-
const events =
|
|
30
|
-
const campaigns =
|
|
31
|
-
const contributions = [...events, ...campaigns];
|
|
49
|
+
const events = processEvents(communityPortal, document);
|
|
50
|
+
const campaigns = processCampaigns(communityPortal, document);
|
|
51
|
+
const contributions = ensureUniqueContributions([...events, ...campaigns]);
|
|
32
52
|
|
|
33
53
|
debug('Finished gathering contributions');
|
|
34
54
|
return contributions;
|
|
35
55
|
}
|
|
36
56
|
|
|
37
|
-
|
|
57
|
+
function processEvents(communityPortal, document) {
|
|
38
58
|
debug('Getting events from Community Portal');
|
|
39
59
|
const allEventNodes = document.querySelectorAll('.profile__event');
|
|
40
60
|
|
|
41
|
-
|
|
61
|
+
return Array.from(allEventNodes, (entity) =>
|
|
42
62
|
formatEvent(communityPortal.participationType, entity)
|
|
43
63
|
)
|
|
44
64
|
.sort((a, b) => b.createdAt - a.createdAt)
|
|
45
65
|
.filter((event) => event.createdAt < new Date());
|
|
46
|
-
return Array.from(new Set(allEvents));
|
|
47
66
|
}
|
|
48
67
|
|
|
49
|
-
|
|
68
|
+
function processCampaigns(communityPortal, document) {
|
|
50
69
|
debug('Getting campaigns from Community Portal');
|
|
51
70
|
const allCampaignNodes = document.querySelectorAll('.profile__campaign');
|
|
52
71
|
|
|
@@ -58,17 +77,25 @@ async function processCampaigns(communityPortal, document) {
|
|
|
58
77
|
}
|
|
59
78
|
|
|
60
79
|
async function getCommunityPortalDocument(url) {
|
|
61
|
-
|
|
80
|
+
// JSDOM.fromURL has no way to set a timeout, so we fetch the page ourselves.
|
|
81
|
+
const response = await fetch(url, {
|
|
82
|
+
signal: AbortSignal.timeout(FETCH_TIMEOUT),
|
|
83
|
+
});
|
|
84
|
+
if (!response.ok) {
|
|
85
|
+
throw new Error(
|
|
86
|
+
`Community Portal request to ${url} failed with status ${response.status}`
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
const html = await response.text();
|
|
90
|
+
const dom = new JSDOM(html, { url: response.url || url });
|
|
62
91
|
const { document } = dom.window;
|
|
63
92
|
return document;
|
|
64
93
|
}
|
|
65
94
|
|
|
66
95
|
function formatEvent(type, entity) {
|
|
67
96
|
return {
|
|
68
|
-
createdAt:
|
|
69
|
-
entity
|
|
70
|
-
.querySelector('.profile__event-time')
|
|
71
|
-
.textContent.replace(/[\n\t∙]/g, '')
|
|
97
|
+
createdAt: parseEventTime(
|
|
98
|
+
entity.querySelector('.profile__event-time').textContent
|
|
72
99
|
),
|
|
73
100
|
description: entity
|
|
74
101
|
.querySelector('.profile__event-title')
|
|
@@ -80,12 +107,10 @@ function formatEvent(type, entity) {
|
|
|
80
107
|
}
|
|
81
108
|
|
|
82
109
|
function formatCampaign(type, entity) {
|
|
83
|
-
const dateValue = entity
|
|
84
|
-
.querySelector('.profile__campaign-dates')
|
|
85
|
-
.textContent.replace(/[\n\t∙]/g, '')
|
|
86
|
-
.replace(/\s*-\s*[a-zA-Z]+\s\d{2}/g, '');
|
|
87
110
|
return {
|
|
88
|
-
createdAt:
|
|
111
|
+
createdAt: parseCampaignDates(
|
|
112
|
+
entity.querySelector('.profile__campaign-dates').textContent
|
|
113
|
+
),
|
|
89
114
|
description: entity
|
|
90
115
|
.querySelector('.profile__campaign-title')
|
|
91
116
|
.textContent.replace(/[\n\t]/g, ''),
|
|
@@ -94,3 +119,49 @@ function formatCampaign(type, entity) {
|
|
|
94
119
|
source: CAMPAIGNS_CATEGORY,
|
|
95
120
|
};
|
|
96
121
|
}
|
|
122
|
+
|
|
123
|
+
function normalizeWhitespace(text) {
|
|
124
|
+
return text.replace(/\s+/g, ' ').trim();
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function monthIndex(name) {
|
|
128
|
+
return MONTHS.indexOf(name.toLowerCase());
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// The profile renders event times as "August 08, 2022 ∙ 18:00". The page does
|
|
132
|
+
// not state a time zone, so we interpret it as UTC to stay independent of the
|
|
133
|
+
// machine's time zone.
|
|
134
|
+
function parseEventTime(text) {
|
|
135
|
+
const value = normalizeWhitespace(text);
|
|
136
|
+
const match = value.match(
|
|
137
|
+
/^([a-zA-Z]+) (\d{1,2}), (\d{4})\s*∙\s*(\d{1,2}):(\d{2})$/
|
|
138
|
+
);
|
|
139
|
+
const month = match ? monthIndex(match[1]) : -1;
|
|
140
|
+
if (month === -1) {
|
|
141
|
+
throw new Error(`Could not parse Community Portal event time "${value}"`);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const [, , day, year, hours, minutes] = match;
|
|
145
|
+
return new Date(Date.UTC(year, month, day, hours, minutes));
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// The profile renders campaign dates either as "January 01 2022" or as a range
|
|
149
|
+
// "December 01 - January 15 2022", where only the end date carries the year.
|
|
150
|
+
function parseCampaignDates(text) {
|
|
151
|
+
const value = normalizeWhitespace(text);
|
|
152
|
+
const match = value.match(
|
|
153
|
+
/^([a-zA-Z]+) (\d{1,2})(?: - ([a-zA-Z]+) \d{1,2})? (\d{4})$/
|
|
154
|
+
);
|
|
155
|
+
const month = match ? monthIndex(match[1]) : -1;
|
|
156
|
+
const endMonth = match && match[3] ? monthIndex(match[3]) : month;
|
|
157
|
+
if (month === -1 || endMonth === -1) {
|
|
158
|
+
throw new Error(
|
|
159
|
+
`Could not parse Community Portal campaign dates "${value}"`
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const [, , day, , endYear] = match;
|
|
164
|
+
// A range ending in an earlier month than it started spans a year boundary.
|
|
165
|
+
const year = endMonth < month ? endYear - 1 : endYear;
|
|
166
|
+
return new Date(Date.UTC(year, month, day));
|
|
167
|
+
}
|
package/lib/contributions.js
CHANGED
|
@@ -1,19 +1,81 @@
|
|
|
1
|
+
// Existing contributions are read from JSON and therefore have a string as
|
|
2
|
+
// createdAt, while freshly fetched ones have a Date. Normalize before comparing.
|
|
3
|
+
function getContributionKey(contribution) {
|
|
4
|
+
return [
|
|
5
|
+
contribution.source,
|
|
6
|
+
contribution.type,
|
|
7
|
+
contribution.link,
|
|
8
|
+
new Date(contribution.createdAt).getTime(),
|
|
9
|
+
].join('|');
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
// Invalid dates would make the comparator return NaN, which leaves the sort
|
|
13
|
+
// order undefined. Treat them as the oldest possible date instead.
|
|
14
|
+
function getSortTime(contribution) {
|
|
15
|
+
const time = new Date(contribution.createdAt).getTime();
|
|
16
|
+
return Number.isNaN(time) ? -Infinity : time;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// Sorts newest first, with contributions without a valid date at the end.
|
|
20
|
+
// Ties keep their original order, as Array.prototype.sort is stable.
|
|
21
|
+
export const compareContributions = (a, b) => {
|
|
22
|
+
const timeA = getSortTime(a);
|
|
23
|
+
const timeB = getSortTime(b);
|
|
24
|
+
if (timeA === timeB) {
|
|
25
|
+
return 0;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
return timeB > timeA ? 1 : -1;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
// Keeps the first occurrence of each contribution.
|
|
1
32
|
export const ensureUniqueContributions = (allContributions) => {
|
|
2
|
-
const
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
33
|
+
const seen = new Set();
|
|
34
|
+
return allContributions.filter((contribution) => {
|
|
35
|
+
const key = getContributionKey(contribution);
|
|
36
|
+
if (seen.has(key)) {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
seen.add(key);
|
|
41
|
+
return true;
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
// Sort object keys so that the serialization does not depend on key order.
|
|
46
|
+
// Dates are already converted to ISO strings by their toJSON at this point.
|
|
47
|
+
function sortKeys(key, value) {
|
|
48
|
+
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
49
|
+
return Object.fromEntries(
|
|
50
|
+
Object.entries(value).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
9
51
|
);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return value;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Compares the stored content of two contribution lists, ignoring the order
|
|
58
|
+
// of the entries as well as the key order within each entry.
|
|
59
|
+
export const haveSameContributions = (contributionsA, contributionsB) => {
|
|
60
|
+
if (contributionsA.length !== contributionsB.length) {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const counts = new Map();
|
|
65
|
+
for (const contribution of contributionsA) {
|
|
66
|
+
const serialized = JSON.stringify(contribution, sortKeys);
|
|
67
|
+
counts.set(serialized, (counts.get(serialized) || 0) + 1);
|
|
68
|
+
}
|
|
10
69
|
|
|
11
|
-
|
|
12
|
-
|
|
70
|
+
for (const contribution of contributionsB) {
|
|
71
|
+
const serialized = JSON.stringify(contribution, sortKeys);
|
|
72
|
+
const count = counts.get(serialized);
|
|
73
|
+
if (!count) {
|
|
74
|
+
return false;
|
|
13
75
|
}
|
|
14
76
|
|
|
15
|
-
|
|
16
|
-
}
|
|
77
|
+
counts.set(serialized, count - 1);
|
|
78
|
+
}
|
|
17
79
|
|
|
18
|
-
return
|
|
80
|
+
return true;
|
|
19
81
|
};
|