volunteer-contributions-fetch 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,6 +18,7 @@ jobs:
18
18
  with:
19
19
  node-version: 20
20
20
  - run: npm ci
21
+ - run: npm run eslint
21
22
  - run: npm test
22
23
  - name: Release
23
24
  env:
@@ -3,7 +3,7 @@ name: Test
3
3
  on: push
4
4
 
5
5
  jobs:
6
- build:
6
+ test:
7
7
  runs-on: ubuntu-latest
8
8
  steps:
9
9
  - uses: actions/checkout@v4
package/README.md CHANGED
@@ -24,6 +24,8 @@ Currently the following sources are supported. Feel free to contribute a PR if y
24
24
  - Mozilla Community Portal
25
25
  - Participated in event
26
26
  - Participated in campaign
27
+ - OpenStreetMap
28
+ - Changesets
27
29
 
28
30
  Each of the sources can be enabled separately. See the configuration section below.
29
31
 
@@ -165,6 +167,14 @@ This section documents all the possible configuration values. For examples see t
165
167
  | `editType` | string | No | Wiki Edit | Label for entries related to having edited an article. |
166
168
  | `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track |
167
169
 
170
+ #### OpenStreetMaps
171
+
172
+ | Field | Data Type | Required | Default | Description |
173
+ | ------------- | --------- | -------- | ------- | -------------------------------------------------------------------------------------------------------------- |
174
+ | `enabled` | boolean | Yes | - | Indicates whether this data source is enabled. Alternatively you can also leave off the `osm` object entirely. |
175
+ | `displayName` | string | Yes | - | Username on OpenStreetMaps |
176
+ | `stopDate` | string | Yes | - | `new Date()` parseable string containing a date of possibly earliest contribution to track |
177
+
168
178
  ### Sample configuration
169
179
 
170
180
  ```js
@@ -214,6 +224,11 @@ const config = {
214
224
  reviewedPR: 'Reviewed a Pull Request',
215
225
  },
216
226
  },
227
+ osm: {
228
+ enabled: true,
229
+ displayName: 'foo',
230
+ stopDate: '2010-01-01',
231
+ },
217
232
  };
218
233
  ```
219
234
 
@@ -0,0 +1,28 @@
1
+ // These fixtures are minified and do not reflect all real attributes and tags.
2
+
3
+ export const firstPage = `
4
+ <osm version="0.6" generator="OpenStreetMap server" copyright="OpenStreetMap and contributors" attribution="http://www.openstreetmap.org/copyright" license="http://opendatacommons.org/licenses/odbl/1-0/">
5
+ ${Array(100)
6
+ .fill(0)
7
+ .map(
8
+ (_value, index) => `
9
+ <changeset id="${index + 1}" created_at="2024-12-29T12:53:41Z">
10
+ <tag k="comment" v="Some test description ${index + 1}"/>
11
+ <tag k="host" v="https://www.openstreetmap.org/edit"/>
12
+ <tag k="source" v="survey"/>
13
+ </changeset>
14
+ `
15
+ )
16
+ .join()}
17
+ </osm>
18
+ `;
19
+
20
+ export const secondPage = `
21
+ <osm version="0.6" generator="OpenStreetMap server" copyright="OpenStreetMap and contributors" attribution="http://www.openstreetmap.org/copyright" license="http://opendatacommons.org/licenses/odbl/1-0/">
22
+ <changeset id="101" created_at="2024-12-29T12:53:41Z">
23
+ <tag k="comment" v="Some test description 101"/>
24
+ <tag k="host" v="https://www.openstreetmap.org/edit"/>
25
+ <tag k="source" v="survey"/>
26
+ </changeset>
27
+ </osm>
28
+ `;
package/lib/fetch.js CHANGED
@@ -5,6 +5,7 @@ import * as bugzilla from './bugzilla.js';
5
5
  import * as communityPortal from './community-portal.js';
6
6
  import * as discourse from './discourse.js';
7
7
  import * as github from './github.js';
8
+ import * as osm from './osm.js';
8
9
  import { validateConfig } from './validator.js';
9
10
  import * as wiki from './wiki.js';
10
11
 
@@ -35,6 +36,7 @@ export async function fetchAll(config, contributions = []) {
35
36
  const wikiResult = await wiki.gather(config);
36
37
  const communityPortalResult = await communityPortal.gather(config);
37
38
  const discourseResult = await discourse.gather(config, existingContributions);
39
+ const osmResult = await osm.gather(config);
38
40
 
39
41
  // Some types are fetched on every update run, therefore we want to filter them out for now.
40
42
  // Otherwise we might end up with duplicate entries we'd need to filter.
@@ -48,6 +50,7 @@ export async function fetchAll(config, contributions = []) {
48
50
  discourse.POSTS_CATEGORY,
49
51
  discourse.TOPICS_CATEGORY,
50
52
  wiki.WIKI_CATEGORY,
53
+ osm.OSM_CATEGORY,
51
54
  ];
52
55
  const existingContributionsToSave = existingContributions.filter(
53
56
  (activity) => {
@@ -61,6 +64,7 @@ export async function fetchAll(config, contributions = []) {
61
64
  debug('COMMUNITY_PORTAL_CONTRIBUTIONS', communityPortalResult.length);
62
65
  debug('DISCOURSE_CONTRIBUTIONS', discourseResult.length);
63
66
  debug('WIKI_CONTRIBUTIONS', wikiResult.length);
67
+ debug('OSM_CONTRIBUTIONS', osmResult.length);
64
68
 
65
69
  const uniqueContributions = Array.from(
66
70
  new Set([
@@ -68,6 +72,7 @@ export async function fetchAll(config, contributions = []) {
68
72
  ...discourseResult, // always a new fetch (previous entries removed above)
69
73
  ...wikiResult, // always a new fetch (previous entries removed above)
70
74
  ...communityPortalResult, // always a new fetch (previous entries removed above)
75
+ ...osmResult, // always a new fetch (previous entries removed above)
71
76
  ...githubResult,
72
77
  ...bugzillaResult,
73
78
  ])
package/lib/fetch.test.js CHANGED
@@ -4,6 +4,7 @@ import { afterEach, beforeEach, expect, test, vi } from 'vitest';
4
4
  import * as communityPortal from './community-portal.js';
5
5
  import * as discourse from './discourse.js';
6
6
  import * as github from './github.js';
7
+ import * as osm from './osm.js';
7
8
  import * as wiki from './wiki.js';
8
9
 
9
10
  import * as validator from './validator.js';
@@ -24,6 +25,7 @@ beforeEach(() => {
24
25
  vi.spyOn(communityPortal, 'gather').mockReturnValue([]);
25
26
  vi.spyOn(discourse, 'gather').mockReturnValue([]);
26
27
  vi.spyOn(github, 'gather').mockReturnValue([]);
28
+ vi.spyOn(osm, 'gather').mockReturnValue([]);
27
29
  vi.spyOn(wiki, 'gather').mockReturnValue([]);
28
30
  });
29
31
 
@@ -79,12 +81,13 @@ test('should take all return values', async () => {
79
81
  ]);
80
82
  discourse.gather.mockResolvedValueOnce([{ source: 'discourse' }]);
81
83
  github.gather.mockResolvedValueOnce([{ source: 'github' }]);
84
+ osm.gather.mockResolvedValueOnce([{ source: 'osm' }]);
82
85
  wiki.gather.mockResolvedValueOnce([{ source: 'wiki' }]);
83
86
 
84
87
  await fetchAll(config);
85
88
  expect(fs.writeFile).toHaveBeenCalledOnce();
86
89
 
87
- expect(originalJSONParse(fs.writeFile.mock.calls[0][1]).length).toBe(4);
90
+ expect(originalJSONParse(fs.writeFile.mock.calls[0][1]).length).toBe(5);
88
91
  });
89
92
 
90
93
  test('should return contributions', async () => {
@@ -1,4 +1,5 @@
1
1
  import Debug from 'debug';
2
+ // eslint-disable-next-line import/no-unresolved
2
3
  import { Octokit } from '@octokit/rest';
3
4
 
4
5
  const debug = Debug('contributions:github-fetcher');
package/lib/osm.js ADDED
@@ -0,0 +1,100 @@
1
+ import Debug from 'debug';
2
+ import { parseString } from 'xml2js';
3
+ import { promisify } from 'util';
4
+
5
+ const debug = Debug('contributions:openstreetmaps');
6
+ const parseStringAsync = promisify(parseString);
7
+
8
+ export const OSM_CATEGORY = 'osm';
9
+
10
+ const TYPE_EDIT = 'OpenStreetMaps Edit';
11
+
12
+ const MAX_EDITS_PER_PAGE = 100;
13
+
14
+ export function validate(config) {
15
+ debug('Validating OSM config');
16
+
17
+ if (!config) {
18
+ return;
19
+ }
20
+
21
+ if (config && config.enabled && !config.displayName) {
22
+ throw new Error('OSM: displayName is required');
23
+ }
24
+
25
+ if (config && config.enabled && !config.stopDate) {
26
+ throw new Error('OSM: stopDate is required');
27
+ }
28
+ }
29
+
30
+ export async function gather({ osm }) {
31
+ if (!osm || (osm && !osm.enabled)) {
32
+ debug('OSM source not enabled, skipping');
33
+ return [];
34
+ }
35
+
36
+ const edits = await processEdits(osm);
37
+
38
+ debug('Finished gathering contributions');
39
+ return edits;
40
+ }
41
+
42
+ async function processEdits({ displayName, stopDate }) {
43
+ debug('Getting edits from OSM');
44
+ let allEdits = [];
45
+ let hasMoreData = true;
46
+ let createdBefore = new Date().toISOString();
47
+
48
+ while (hasMoreData) {
49
+ // eslint-disable-next-line no-await-in-loop
50
+ const edits = await getPage(displayName, stopDate, createdBefore);
51
+ debug(`Got ${edits.length} results to filter`);
52
+ const processed = processEntities(edits);
53
+ debug(`Got ${processed.length} results`);
54
+ allEdits = allEdits.concat(processed);
55
+ hasMoreData = edits.length === MAX_EDITS_PER_PAGE;
56
+ createdBefore = processed[processed.length - 1].createdAt;
57
+ }
58
+
59
+ return allEdits;
60
+ }
61
+
62
+ function getUrl({ displayName, createdBefore, stopDate }) {
63
+ return `https://api.openstreetmap.org/api/0.6/changesets?display_name=${displayName}&time=${stopDate},${createdBefore}`;
64
+ }
65
+
66
+ async function getPage(displayName, stopDate, createdBefore) {
67
+ const url = getUrl({
68
+ displayName,
69
+ stopDate,
70
+ createdBefore,
71
+ });
72
+ debug(`Getting ${url}`);
73
+ const response = await fetch(url);
74
+ const data = await response.text();
75
+ const parsed = await parse(data);
76
+ const items =
77
+ parsed.osm && Array.isArray(parsed.osm.changeset)
78
+ ? parsed.osm.changeset
79
+ : [];
80
+ return items;
81
+ }
82
+
83
+ function processEntities(entities) {
84
+ return entities.map((entity) => format(entity));
85
+ }
86
+
87
+ function format(entity) {
88
+ return {
89
+ createdAt: new Date(entity.$.created_at),
90
+ description: entity.tag.find((tag) => tag.$.k === 'comment').$.v,
91
+ link: `https://www.openstreetmap.org/changeset/${entity.$.id}`,
92
+ type: TYPE_EDIT,
93
+ source: OSM_CATEGORY,
94
+ };
95
+ }
96
+
97
+ async function parse(xml) {
98
+ const parsed = await parseStringAsync(xml);
99
+ return parsed;
100
+ }
@@ -0,0 +1,97 @@
1
+ import { afterEach, beforeEach, expect, test, vi } from 'vitest';
2
+
3
+ import { firstPage, secondPage } from '../fixtures/osm.js';
4
+ import * as osm from './osm.js';
5
+
6
+ const config = {
7
+ osm: {
8
+ enabled: true,
9
+ displayName: 'user',
10
+ stopDate: new Date('2021-11-01'),
11
+ },
12
+ };
13
+
14
+ beforeEach(() => {
15
+ const mockedFetch = vi
16
+ .fn()
17
+ .mockResolvedValueOnce({ text: vi.fn().mockResolvedValue(firstPage) })
18
+ .mockResolvedValueOnce({ text: vi.fn().mockResolvedValue(secondPage) });
19
+ vi.stubGlobal('fetch', mockedFetch);
20
+ });
21
+
22
+ afterEach(() => {
23
+ vi.restoreAllMocks();
24
+ });
25
+
26
+ test('should validate config - missing completely', () => {
27
+ const emptyConfig = {};
28
+ expect(() => osm.validate(emptyConfig)).not.toThrow();
29
+ });
30
+
31
+ test('should validate config - missing displayName', () => {
32
+ const failConfig = {
33
+ enabled: true,
34
+ stopDate: new Date(),
35
+ };
36
+
37
+ expect(() => osm.validate(failConfig)).toThrow();
38
+ });
39
+
40
+ test('should validate config - missing stopDate', () => {
41
+ const failConfig = {
42
+ enabled: true,
43
+ displayName: 'user',
44
+ };
45
+
46
+ expect(() => osm.validate(failConfig)).toThrow();
47
+ });
48
+
49
+ test('should fetch', async () => {
50
+ const contributions = await osm.gather(config);
51
+ expect(contributions.length).toBe(101);
52
+ });
53
+
54
+ test('should not fetch if disabled - by enable property', async () => {
55
+ const specificTypeConfig = {
56
+ osm: {
57
+ ...config.osm,
58
+ enabled: false,
59
+ },
60
+ };
61
+
62
+ const contributions = await osm.gather(specificTypeConfig);
63
+
64
+ expect(contributions.length).toBe(0);
65
+ });
66
+
67
+ test('should not fetch if disabled - by leaving off config value', async () => {
68
+ const contributions = await osm.gather({});
69
+
70
+ expect(contributions.length).toBe(0);
71
+ });
72
+
73
+ test('should format', async () => {
74
+ const contributions = await osm.gather(config);
75
+
76
+ expect(contributions[0]).toStrictEqual({
77
+ createdAt: new Date('2024-12-29T12:53:41.000Z'),
78
+ description: 'Some test description 1',
79
+ link: 'https://www.openstreetmap.org/changeset/1',
80
+ source: 'osm',
81
+ type: 'OpenStreetMaps Edit',
82
+ });
83
+ expect(contributions[1]).toStrictEqual({
84
+ createdAt: new Date('2024-12-29T12:53:41.000Z'),
85
+ description: 'Some test description 2',
86
+ link: 'https://www.openstreetmap.org/changeset/2',
87
+ source: 'osm',
88
+ type: 'OpenStreetMaps Edit',
89
+ });
90
+ expect(contributions[100]).toStrictEqual({
91
+ createdAt: new Date('2024-12-29T12:53:41.000Z'),
92
+ description: 'Some test description 101',
93
+ link: 'https://www.openstreetmap.org/changeset/101',
94
+ source: 'osm',
95
+ type: 'OpenStreetMaps Edit',
96
+ });
97
+ });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "volunteer-contributions-fetch",
3
- "version": "3.0.0",
3
+ "version": "3.1.0",
4
4
  "description": "Library to fetch volunteer contributions from several sources such as GitHub",
5
5
  "author": "Michael Kohler",
6
6
  "license": "MIT",