volunteer-contributions-fetch 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.editorconfig +13 -0
- package/.eslintrc +18 -0
- package/.github/workflows/release.yml +22 -0
- package/.github/workflows/test.yml +21 -0
- package/.husky/commit-msg +4 -0
- package/LICENSE +10 -0
- package/README.md +194 -0
- package/commitlint.config.js +1 -0
- package/index.js +5 -0
- package/lib/community-portal.js +107 -0
- package/lib/discourse.js +144 -0
- package/lib/discourse.test.js +106 -0
- package/lib/fetch.js +62 -0
- package/lib/github-fetcher.js +62 -0
- package/lib/github-fetcher.test.js +6 -0
- package/lib/github.js +435 -0
- package/lib/validator.js +26 -0
- package/lib/wiki.js +150 -0
- package/package.json +62 -0
package/.editorconfig
ADDED
package/.eslintrc
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": "airbnb-base",
|
|
3
|
+
"settings": {
|
|
4
|
+
"import/resolver": {
|
|
5
|
+
"node": {
|
|
6
|
+
"paths": ["lib"],
|
|
7
|
+
"extensions": [".js", ".ts", ".d.ts", ".tsx"]
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
},
|
|
11
|
+
"rules": {
|
|
12
|
+
"arrow-body-style": 0,
|
|
13
|
+
"comma-dangle": 0,
|
|
14
|
+
"no-plusplus": 0,
|
|
15
|
+
"no-restricted-syntax": 0,
|
|
16
|
+
"no-use-before-define": 0
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
release:
|
|
10
|
+
runs-on: ubuntu-22.04
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@9a3a9ade8222dcdf9d3c77710b10df47ee7c7c89
|
|
13
|
+
- uses: actions/setup-node@v1
|
|
14
|
+
with:
|
|
15
|
+
node-version: 16
|
|
16
|
+
- run: npm ci
|
|
17
|
+
- run: npm test
|
|
18
|
+
- name: Release
|
|
19
|
+
env:
|
|
20
|
+
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
|
21
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
22
|
+
run: npx semantic-release
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
name: Test
|
|
2
|
+
|
|
3
|
+
on: push
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
runs-on: ubuntu-latest
|
|
8
|
+
steps:
|
|
9
|
+
- uses: actions/checkout@9a3a9ade8222dcdf9d3c77710b10df47ee7c7c89
|
|
10
|
+
with:
|
|
11
|
+
submodules: 'recursive'
|
|
12
|
+
- name: Use Node.js 16
|
|
13
|
+
uses: actions/setup-node@v1
|
|
14
|
+
with:
|
|
15
|
+
node-version: 16
|
|
16
|
+
- name: Install Dependencies
|
|
17
|
+
run: npm ci
|
|
18
|
+
- name: Linting
|
|
19
|
+
run: npm run eslint
|
|
20
|
+
- name: Test
|
|
21
|
+
run: npm test
|
package/LICENSE
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
|
|
2
|
+
The MIT License (MIT)
|
|
3
|
+
|
|
4
|
+
Copyright (c) 2022 Michael Kohler
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
7
|
+
|
|
8
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
9
|
+
|
|
10
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# volunteer-contributions-fetch
|
|
2
|
+
|
|
3
|
+
This library fetches contributions from several different sources and saves them in a JSON file. This can be used to track your own volunteer contributions. An example of this can be found on [Michael's website](https://mkohler.dev/contributions/).
|
|
4
|
+
|
|
5
|
+
## Sources
|
|
6
|
+
|
|
7
|
+
### Supported Sources
|
|
8
|
+
|
|
9
|
+
Currently the following sources are supported. Feel free to contribute a PR if you'd like to support another source!
|
|
10
|
+
|
|
11
|
+
* GitHub
|
|
12
|
+
* Issues
|
|
13
|
+
* PRs
|
|
14
|
+
* PR Reviews
|
|
15
|
+
* Wiki
|
|
16
|
+
* Page edited
|
|
17
|
+
* Discourse
|
|
18
|
+
* Topic created
|
|
19
|
+
* Post created
|
|
20
|
+
* Mozilla Community Portal
|
|
21
|
+
* Participated in event
|
|
22
|
+
* Participated in campaign
|
|
23
|
+
|
|
24
|
+
Each of the sources can be enabled separately. See the configuration section below.
|
|
25
|
+
|
|
26
|
+
### Planned Sources
|
|
27
|
+
|
|
28
|
+
The following sources are planned:
|
|
29
|
+
|
|
30
|
+
* GitHub
|
|
31
|
+
* Comments
|
|
32
|
+
* Bugzilla
|
|
33
|
+
* Bug reported
|
|
34
|
+
* Bug commented on
|
|
35
|
+
* Phabricator
|
|
36
|
+
* Revision created
|
|
37
|
+
* Revision updated
|
|
38
|
+
|
|
39
|
+
### Possible future sources
|
|
40
|
+
|
|
41
|
+
The following sources would be interesting for (at least) Mozilla contributors:
|
|
42
|
+
|
|
43
|
+
* support.mozilla.org
|
|
44
|
+
* Pontoon
|
|
45
|
+
|
|
46
|
+
## Usage
|
|
47
|
+
|
|
48
|
+
First install the package from npm:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
npm install volunteer-contribution-fetch
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Then you can create a file in your project which runs the fetch operation and provides the necesary config:
|
|
55
|
+
|
|
56
|
+
```js
|
|
57
|
+
const { fetchAll } = require('volunteer-contributions-fetch');
|
|
58
|
+
|
|
59
|
+
const config = {}; // See below for documentation and a sample config
|
|
60
|
+
|
|
61
|
+
fetchAll(config);
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Configuration
|
|
65
|
+
|
|
66
|
+
You can configure the different providers with a config you pass into the `fetchAll` function call. These config parameters are outlined below. The config gets validated and you might get an error if you haven't specified all necessary properties.
|
|
67
|
+
|
|
68
|
+
Additionally to the configuration file, we also require some environment variables to be set:
|
|
69
|
+
|
|
70
|
+
* `GITHUB_TOKEN` with a GitHub token, if the GitHub source is enabled
|
|
71
|
+
|
|
72
|
+
While these tokens could potentially also be included the config, we want to make it hard to mistakenly publish secrets within code.
|
|
73
|
+
|
|
74
|
+
### Configuration documentation
|
|
75
|
+
|
|
76
|
+
This section documents all the possible configuration values. For examples see the sample configuration below.
|
|
77
|
+
|
|
78
|
+
#### General
|
|
79
|
+
|
|
80
|
+
| Field | Data Type | Required | Default | Description |
|
|
81
|
+
|-------------------|-----------|----------|---------|----------------------------------------------------|
|
|
82
|
+
| `outputFile` | string | Yes | - | Path to contributions output file |
|
|
83
|
+
| `communityPortal` | Object | No | - | Community Portal configuration options (see below) |
|
|
84
|
+
| `discourse` | Object | No | - | Discourse configuration options (see below) |
|
|
85
|
+
| `github` | Object | No | - | GitHub configuration options (see below) |
|
|
86
|
+
| `wiki` | Object | No | - | Wiki configuration options (see below) |
|
|
87
|
+
|
|
88
|
+
#### Community Portal
|
|
89
|
+
|
|
90
|
+
| Field | Data Type | Required | Default | Description |
|
|
91
|
+
|---------------------|-----------|----------|---------|----------------------------------------------------------------------------------------------------------------------------|
|
|
92
|
+
| `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `communityPortal` object entirely. |
|
|
93
|
+
| `baseUrl` | string | Yes | - | Community Portal base url without trailing slash |
|
|
94
|
+
| `username` | string | Yes | - | Username on the Community Portal |
|
|
95
|
+
| `participationType` | string | Yes | - | Label for entries related to having participated in an event. |
|
|
96
|
+
| `campaignType` | string | Yes | - | Label for entries related to having participated in a campaign |
|
|
97
|
+
|
|
98
|
+
#### Discourse
|
|
99
|
+
|
|
100
|
+
| Field | Data Type | Required | Default | Description |
|
|
101
|
+
|---------------------|-----------|----------|---------|----------------------------------------------------------------------------------------------------------------------------|
|
|
102
|
+
| `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `discourse` object entirely. |
|
|
103
|
+
| `baseUrl` | string | Yes | - | API base url without trailing slash |
|
|
104
|
+
| `username` | string | Yes | - | Username on Discoruse |
|
|
105
|
+
| `topicType` | string | Yes | - | Label for entries related to having created a new topic. |
|
|
106
|
+
| `postType` | string | Yes | - | Label for entries related to having posted on an existing topic. |
|
|
107
|
+
|
|
108
|
+
#### GitHub
|
|
109
|
+
|
|
110
|
+
| Field | Data Type | Required | Default | Description |
|
|
111
|
+
|----------------------------|-----------|----------|---------|----------------------------------------------------------------------------------------------------------------------------|
|
|
112
|
+
| `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `github` object entirely. |
|
|
113
|
+
| `username` | string | Yes | - | Username on GitHub |
|
|
114
|
+
| `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track |
|
|
115
|
+
| `filter` | string | Yes | - | Regex to apply to each entry to filter for organization/repo names. For example: `"mozilla|firefox"` to only include orgs and repos with either "mozilla" or "firefox" it their name.|
|
|
116
|
+
| `types` | Object | Yes | - | Object containing the different labels below |
|
|
117
|
+
| `types.commit` | string | Yes | - | Label for entries related to GitHub commits |
|
|
118
|
+
| `types.privateCommit` | string | Yes | - | Label for entries related to private GitHub commits |
|
|
119
|
+
| `types.createdPR` | string | Yes | - | Label for entries related to opening GitHub PRs |
|
|
120
|
+
| `types.createdIssue` | string | Yes | - | Label for entries related to creating GitHub issues |
|
|
121
|
+
| `types.commentedPR` | string | Yes | - | Label for entries related to commenting on a GitHub PR (review) |
|
|
122
|
+
| `types.approvedPR` | string | Yes | - | Label for entries related to approving a GitHub PR |
|
|
123
|
+
| `types.changesRequestedPR` | string | Yes | - | Label for entries related to requesting changes on a GitHub PR |
|
|
124
|
+
| `types.reviewedPR` | string | Yes | - | Label for entries related to reviewing a GitHub PR (fallback) |
|
|
125
|
+
|
|
126
|
+
### Sample configuration
|
|
127
|
+
|
|
128
|
+
```js
|
|
129
|
+
const config = {
|
|
130
|
+
outputFile: `${__dirname}/_data/contributions.json`,
|
|
131
|
+
communityPortal: {
|
|
132
|
+
enabled: true,
|
|
133
|
+
baseUrl: 'https://community.mozilla.org/people',
|
|
134
|
+
username: 'mkohler',
|
|
135
|
+
participationType: 'Participated in an event',
|
|
136
|
+
campaignType: 'Participated in a campaign',
|
|
137
|
+
},
|
|
138
|
+
discourse: {
|
|
139
|
+
enabled: true,
|
|
140
|
+
baseUrl: 'https://discourse.mozilla.org',
|
|
141
|
+
username: 'mkohler',
|
|
142
|
+
topicType: 'Created Discourse Topic',
|
|
143
|
+
postType: 'Posted on Discourse Topic',
|
|
144
|
+
},
|
|
145
|
+
wiki: {
|
|
146
|
+
enabled: true,
|
|
147
|
+
baseUrl: 'https://wiki.mozilla.org/api.php?action=feedcontributions',
|
|
148
|
+
username: 'michaelkohler',
|
|
149
|
+
editType: 'Wiki Edit',
|
|
150
|
+
stopDate: '2012-05-01',
|
|
151
|
+
},
|
|
152
|
+
github: {
|
|
153
|
+
enabled: true,
|
|
154
|
+
username: 'MichaelKohler',
|
|
155
|
+
stopDate: '2008-01-01',
|
|
156
|
+
filter: 'mozilla|common-voice|reps',
|
|
157
|
+
types: {
|
|
158
|
+
commit: 'GitHub Commit',
|
|
159
|
+
privateCommit: 'Commit in private repository',
|
|
160
|
+
createdPR: 'Created PR',
|
|
161
|
+
createdIssue: 'Created Issue Report',
|
|
162
|
+
commentedPR: 'Commented on a Pull Request',
|
|
163
|
+
approvedPR: 'Approved a Pull Request',
|
|
164
|
+
changesRequestedPR: 'Requested changes on a Pull Request',
|
|
165
|
+
reviewedPR: 'Reviewed a Pull Request',
|
|
166
|
+
}
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
## Developing
|
|
172
|
+
|
|
173
|
+
Any issues or PRs are welcome! Before doing major changes, please create an issue elaborating on the approach.
|
|
174
|
+
|
|
175
|
+
### Setup
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
npm ci
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
### Tests
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
npm run eslint
|
|
185
|
+
npm test
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
### Testing changes
|
|
189
|
+
|
|
190
|
+
You can also add a temporary file with a test config as well as the call to `fetchAll` if you would like to test changes within this project. A sample config can be found in the configuration section above. Please make sure to not include that config file when committing!
|
|
191
|
+
|
|
192
|
+
### Deployments
|
|
193
|
+
|
|
194
|
+
Please use [Angular Commit Message Conventions](https://github.com/angular/angular/blob/master/CONTRIBUTING.md#-commit-message-format) when contributing to this project, as depending on the commit message the GitHub Action will directly publish a new version of the package.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports = { extends: ['@commitlint/config-angular'] };
|
package/index.js
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
const debug = require('debug')('contributions:community-portal');
|
|
2
|
+
const { JSDOM } = require('jsdom');
|
|
3
|
+
|
|
4
|
+
const EVENTS_CATEGORY = 'community-portal-events';
|
|
5
|
+
const CAMPAIGNS_CATEGORY = 'community-portal-campaigns';
|
|
6
|
+
|
|
7
|
+
module.exports = {
|
|
8
|
+
EVENTS_CATEGORY,
|
|
9
|
+
CAMPAIGNS_CATEGORY,
|
|
10
|
+
gather,
|
|
11
|
+
validate,
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
function validate(config) {
|
|
15
|
+
debug('Validating Community Portal config');
|
|
16
|
+
|
|
17
|
+
if (!config) {
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
if (config && config.enabled && !config.baseUrl) {
|
|
22
|
+
throw new Error('Community Portal: baseUrl is required');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
if (config && config.enabled && !config.username) {
|
|
26
|
+
throw new Error('Community Portal: username is required');
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
if (config && config.enabled && !config.participationType) {
|
|
30
|
+
throw new Error('Community Portal: participationType is required');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
if (config && config.enabled && !config.campaignType) {
|
|
34
|
+
throw new Error('Community Portal: campaignType is required');
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function gather({ communityPortal }) {
|
|
39
|
+
if (!communityPortal || (communityPortal && !communityPortal.enabled)) {
|
|
40
|
+
debug('Community Portal source not enabled, skipping');
|
|
41
|
+
return [];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const profileUrl = `${communityPortal.baseUrl}/${communityPortal.username}`;
|
|
45
|
+
const document = await getCommunityPortalDocument(profileUrl);
|
|
46
|
+
const events = await processEvents(communityPortal, document);
|
|
47
|
+
const campaigns = await processCampaigns(communityPortal, document);
|
|
48
|
+
const contributions = [
|
|
49
|
+
...events,
|
|
50
|
+
...campaigns,
|
|
51
|
+
];
|
|
52
|
+
|
|
53
|
+
debug('Finished gathering contributions');
|
|
54
|
+
return contributions;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function processEvents(communityPortal, document) {
|
|
58
|
+
debug('Getting events from Community Portal');
|
|
59
|
+
const allEventNodes = document.querySelectorAll('.profile__event');
|
|
60
|
+
|
|
61
|
+
const allEvents = Array.from(
|
|
62
|
+
allEventNodes,
|
|
63
|
+
(entity) => formatEvent(communityPortal.participationType, entity)
|
|
64
|
+
)
|
|
65
|
+
.sort((a, b) => b.createdAt - a.createdAt)
|
|
66
|
+
.filter((event) => event.createdAt < new Date());
|
|
67
|
+
return Array.from(new Set(allEvents));
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async function processCampaigns(communityPortal, document) {
|
|
71
|
+
debug('Getting campaigns from Community Portal');
|
|
72
|
+
const allCampaignNodes = document.querySelectorAll('.profile__campaign');
|
|
73
|
+
|
|
74
|
+
return Array.from(
|
|
75
|
+
allCampaignNodes,
|
|
76
|
+
(entity) => formatCampaign(communityPortal.campaignType, entity)
|
|
77
|
+
)
|
|
78
|
+
.sort((a, b) => b.createdAt - a.createdAt)
|
|
79
|
+
.filter((campaign) => campaign.createdAt < new Date());
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async function getCommunityPortalDocument(url) {
|
|
83
|
+
const dom = await JSDOM.fromURL(url);
|
|
84
|
+
const { document } = dom.window;
|
|
85
|
+
return document;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function formatEvent(type, entity) {
|
|
89
|
+
return {
|
|
90
|
+
createdAt: new Date(entity.querySelector('.profile__event-time').textContent.replace(/[\n\t∙]/g, '')),
|
|
91
|
+
description: entity.querySelector('.profile__event-title').textContent.replace(/[\n\t]/g, ''),
|
|
92
|
+
link: entity.getAttribute('href'),
|
|
93
|
+
type,
|
|
94
|
+
source: EVENTS_CATEGORY,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function formatCampaign(type, entity) {
|
|
99
|
+
const dateValue = entity.querySelector('.profile__campaign-dates').textContent.replace(/[\n\t∙]/g, '').replace(/\s*-\s*[a-zA-Z]+\s\d{2}/g, '');
|
|
100
|
+
return {
|
|
101
|
+
createdAt: new Date(dateValue),
|
|
102
|
+
description: entity.querySelector('.profile__campaign-title').textContent.replace(/[\n\t]/g, ''),
|
|
103
|
+
link: entity.getAttribute('href'),
|
|
104
|
+
type,
|
|
105
|
+
source: CAMPAIGNS_CATEGORY,
|
|
106
|
+
};
|
|
107
|
+
}
|
package/lib/discourse.js
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
const debug = require('debug')('contributions:discourse');
|
|
2
|
+
const axios = require('axios');
|
|
3
|
+
|
|
4
|
+
const RESULTS_PER_PAGE = 30;
|
|
5
|
+
const TOPICS_CATEGORY = 'discourse-topics';
|
|
6
|
+
const POSTS_CATEGORY = 'discourse-posts';
|
|
7
|
+
|
|
8
|
+
module.exports = {
|
|
9
|
+
TOPICS_CATEGORY,
|
|
10
|
+
POSTS_CATEGORY,
|
|
11
|
+
gather,
|
|
12
|
+
validate,
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
function validate(config) {
|
|
16
|
+
debug('Validating Discourse config');
|
|
17
|
+
|
|
18
|
+
if (!config) {
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
if (config && config.enabled && !config.baseUrl) {
|
|
23
|
+
throw new Error('Discourse: baseUrl is required');
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
if (config && config.enabled && !config.username) {
|
|
27
|
+
throw new Error('Discourse: username is required');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
if (config && config.enabled && !config.topicType) {
|
|
31
|
+
throw new Error('Discourse: topicType is required');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
if (config && config.enabled && !config.postType) {
|
|
35
|
+
throw new Error('Discourse: postType is required');
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function gather({ discourse }) {
|
|
40
|
+
if (!discourse || (discourse && !discourse.enabled)) {
|
|
41
|
+
debug('Discourse source not enabled, skipping');
|
|
42
|
+
return [];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const topics = await processTopics(discourse);
|
|
46
|
+
const posts = await processPosts(discourse);
|
|
47
|
+
const contributions = [
|
|
48
|
+
...topics,
|
|
49
|
+
...posts,
|
|
50
|
+
];
|
|
51
|
+
|
|
52
|
+
debug('Finished gathering contributions');
|
|
53
|
+
return contributions;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function processTopics({ baseUrl, username, topicType }) {
|
|
57
|
+
debug('Getting topics from Discourse');
|
|
58
|
+
let allTopics = [];
|
|
59
|
+
|
|
60
|
+
const page = topicPageGenerator(baseUrl, username);
|
|
61
|
+
|
|
62
|
+
for await (const topicsResponse of page) {
|
|
63
|
+
const { topics } = topicsResponse.topic_list;
|
|
64
|
+
debug(`Got ${topics.length} topics`);
|
|
65
|
+
allTopics = allTopics.concat(topics.map((entity) => formatTopic(topicType, baseUrl, entity)));
|
|
66
|
+
|
|
67
|
+
const hasMore = topics.length === RESULTS_PER_PAGE;
|
|
68
|
+
if (!hasMore) {
|
|
69
|
+
debug('We are done!');
|
|
70
|
+
return allTopics;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function processPosts({ baseUrl, username, postType }) {
|
|
78
|
+
debug('Getting posts from Discourse');
|
|
79
|
+
let allPosts = [];
|
|
80
|
+
|
|
81
|
+
const page = postPageGenerator(baseUrl, username);
|
|
82
|
+
|
|
83
|
+
for await (const postsResponse of page) {
|
|
84
|
+
const posts = postsResponse.user_actions;
|
|
85
|
+
debug(`Got ${posts.length} posts`);
|
|
86
|
+
allPosts = allPosts.concat(posts.map((entity) => formatPost(postType, baseUrl, entity)));
|
|
87
|
+
|
|
88
|
+
const hasMore = posts.length === RESULTS_PER_PAGE;
|
|
89
|
+
if (!hasMore) {
|
|
90
|
+
debug('We are done!');
|
|
91
|
+
return allPosts;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
return undefined;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function* topicPageGenerator(baseUrl, username) {
|
|
99
|
+
let page = 0;
|
|
100
|
+
while (true) {
|
|
101
|
+
debug(`Getting page ${page}`);
|
|
102
|
+
const url = `${baseUrl}/topics/created-by/${username}.json?page=${page}`;
|
|
103
|
+
const results = await getDiscourseData(url); // eslint-disable-line no-await-in-loop
|
|
104
|
+
page++;
|
|
105
|
+
yield results;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async function* postPageGenerator(baseUrl, username) {
|
|
110
|
+
let page = 0;
|
|
111
|
+
while (true) {
|
|
112
|
+
debug(`Getting page ${page}`);
|
|
113
|
+
const offset = page * RESULTS_PER_PAGE;
|
|
114
|
+
const url = `${baseUrl}/user_actions.json?username=${username}&filter=5&offset=${offset}`;
|
|
115
|
+
const results = await getDiscourseData(url); // eslint-disable-line no-await-in-loop
|
|
116
|
+
page++;
|
|
117
|
+
yield results;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async function getDiscourseData(url) {
|
|
122
|
+
const response = await axios.get(url);
|
|
123
|
+
return response.data;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function formatTopic(type, baseUrl, entity) {
|
|
127
|
+
return {
|
|
128
|
+
createdAt: new Date(entity.created_at),
|
|
129
|
+
description: entity.title,
|
|
130
|
+
link: `${baseUrl}/t/${entity.slug}/${entity.id}`,
|
|
131
|
+
type,
|
|
132
|
+
source: TOPICS_CATEGORY,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function formatPost(type, baseUrl, entity) {
|
|
137
|
+
return {
|
|
138
|
+
createdAt: new Date(entity.created_at),
|
|
139
|
+
description: entity.title,
|
|
140
|
+
link: `${baseUrl}/t/${entity.slug}/${entity.topic_id}/${entity.post_number}`,
|
|
141
|
+
type,
|
|
142
|
+
source: POSTS_CATEGORY,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/* eslint camelcase: 0 */
|
|
2
|
+
|
|
3
|
+
const test = require('ava');
|
|
4
|
+
const axios = require('axios');
|
|
5
|
+
const sinon = require('sinon');
|
|
6
|
+
|
|
7
|
+
const discourse = require('./discourse');
|
|
8
|
+
|
|
9
|
+
const TOPICS = {
|
|
10
|
+
topic_list: {
|
|
11
|
+
// eslint-disable-next-line newline-per-chained-call
|
|
12
|
+
topics: new Array(30).fill(0).map((_, index) => ({
|
|
13
|
+
created_at: new Date('2020-04-13'),
|
|
14
|
+
title: `Topic ${index}`,
|
|
15
|
+
slug: `slug-${index}`,
|
|
16
|
+
id: index,
|
|
17
|
+
})),
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
const TOPICS_SMALL = {
|
|
22
|
+
topic_list: {
|
|
23
|
+
topics: [{
|
|
24
|
+
created_at: new Date('2020-04-13'),
|
|
25
|
+
title: 'Topic 1',
|
|
26
|
+
slug: 'slug-1',
|
|
27
|
+
id: 1,
|
|
28
|
+
}],
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
const POSTS = {
|
|
33
|
+
// eslint-disable-next-line newline-per-chained-call
|
|
34
|
+
user_actions: new Array(30).fill(0).map((_, index) => ({
|
|
35
|
+
created_at: new Date('2020-04-13'),
|
|
36
|
+
title: `Post ${index}`,
|
|
37
|
+
slug: `slug-${index}`,
|
|
38
|
+
topic_id: index,
|
|
39
|
+
post_id: 1,
|
|
40
|
+
post_number: 1,
|
|
41
|
+
})),
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
const POSTS_SMALL = {
|
|
45
|
+
user_actions: [{
|
|
46
|
+
created_at: new Date('2020-04-13'),
|
|
47
|
+
title: 'Post 1',
|
|
48
|
+
slug: 'slug-1',
|
|
49
|
+
topic_id: 1,
|
|
50
|
+
post_id: 1,
|
|
51
|
+
post_number: 1,
|
|
52
|
+
}],
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
const config = {
|
|
56
|
+
discourse: {
|
|
57
|
+
enabled: true,
|
|
58
|
+
baseUrl: 'https://discourse.mozilla.org',
|
|
59
|
+
topicType: 'Created Discourse Topic',
|
|
60
|
+
postType: 'Posted on Discourse Topic',
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
test.beforeEach((t) => {
|
|
65
|
+
// eslint-disable-next-line no-param-reassign
|
|
66
|
+
t.context.sandbox = sinon.createSandbox();
|
|
67
|
+
|
|
68
|
+
t.context.sandbox.stub(axios, 'get').onCall(0)
|
|
69
|
+
.resolves({ data: TOPICS });
|
|
70
|
+
axios.get.onCall(1)
|
|
71
|
+
.resolves({ data: TOPICS_SMALL });
|
|
72
|
+
axios.get.onCall(2)
|
|
73
|
+
.resolves({ data: POSTS });
|
|
74
|
+
axios.get.onCall(3)
|
|
75
|
+
.resolves({ data: POSTS_SMALL });
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test.afterEach.always((t) => {
|
|
79
|
+
t.context.sandbox.restore();
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test.serial('should fetch', async (t) => {
|
|
83
|
+
const contributions = await discourse.gather(config);
|
|
84
|
+
t.is(contributions.length, 30 + 1 + 30 + 1);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test.serial('should format', async (t) => {
|
|
88
|
+
const contributions = await discourse.gather(config);
|
|
89
|
+
|
|
90
|
+
// Topic
|
|
91
|
+
t.deepEqual(contributions[0], {
|
|
92
|
+
createdAt: new Date('2020-04-13'),
|
|
93
|
+
description: 'Topic 0',
|
|
94
|
+
link: 'https://discourse.mozilla.org/t/slug-0/0',
|
|
95
|
+
type: 'Created Discourse Topic',
|
|
96
|
+
source: 'discourse-topics',
|
|
97
|
+
});
|
|
98
|
+
// Post
|
|
99
|
+
t.deepEqual(contributions[31], {
|
|
100
|
+
createdAt: new Date('2020-04-13'),
|
|
101
|
+
description: 'Post 0',
|
|
102
|
+
link: 'https://discourse.mozilla.org/t/slug-0/0/1',
|
|
103
|
+
type: 'Posted on Discourse Topic',
|
|
104
|
+
source: 'discourse-posts',
|
|
105
|
+
});
|
|
106
|
+
});
|