website-xp-phone 2.2.1 → 2.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +8 -0
- package/Makefile +5 -1
- package/README.md +9 -0
- package/build/routes.ts +27 -10
- package/package.json +2 -1
- package/public/Click.ogg +0 -0
- package/public/addons/rage-volley.txt +1 -1
- package/public/addons/rueg-wa-druid.txt +1 -1
- package/public/addons/rueg-wa-mage.txt +1 -1
- package/public/addons/rueg-wa-paladin.txt +1 -0
- package/public/addons/rueg-wa-priest.txt +1 -1
- package/public/addons/rueg-wa-rueg.txt +1 -1
- package/public/addons/rueg-wa-shaman.txt +1 -1
- package/public/blog/random/branch.md +29 -0
- package/public/blog/random/google.md +25 -0
- package/public/checksums.txt +70 -47
- package/public/documents/drift-report.json +46 -148
- package/public/loading.css +7 -0
- package/scripts/asset-reference.mjs +37 -0
- package/scripts/mutable-assets.mjs +46 -39
- package/scripts/refresh-soundcloud.mjs +152 -414
- package/scripts/verify-policy.mjs +206 -0
- package/scripts/write-dist-checksums.mjs +251 -135
- package/src/content/addons.json +13 -17
- package/src/content/blog.json +5 -26
- package/src/content/characters.json +6 -6
- package/src/content/configs.json +6 -2
- package/src/content/index.ts +105 -12
- package/src/content/pages.json +10 -1
- package/src/content/projects/development.json +9 -1
- package/src/content/projects/doftiles.json +9 -1
- package/src/content/projects/frontend.json +9 -1
- package/src/content/projects/graphbrowser.json +9 -1
- package/src/content/projects/herbstutil.json +93 -4
- package/src/content/projects/lightdni-jssas-toggle.json +9 -1
- package/src/content/projects/pagerts.json +13 -2
- package/src/content/projects/rarebert.json +12 -2
- package/src/content/projects/webuser.json +9 -1
- package/src/content/schema.ts +10 -1
- package/src/content/sections.json +2 -3
- package/src/content/soundcloud.json +204 -72
- package/src/lib/audio.ts +14 -0
- package/src/lib/clippy.ts +1 -7
- package/src/lib/embeds.ts +1 -1
- package/src/lib/musicSchema.ts +1 -0
- package/src/main.tsx +8 -0
- package/src/pages/MusicPage.tsx +0 -1
- package/src/pages/SitemapPage.tsx +1 -1
- package/src/pages/WowPage.tsx +1 -1
- package/src/shell/App.tsx +4 -1
- package/src/styles/extras.css +85 -30
- package/src/styles/layout.css +27 -18
- package/src/styles/window.css +53 -50
- package/src/themes/pageSlide/index.ts +115 -13
- package/src/themes/pageSlide/pageSlide.css +99 -6
- package/src/window/Embed.tsx +3 -1
- package/src/window/MaximizedWindow.tsx +250 -0
- package/src/window/OkButton.tsx +6 -1
- package/src/window/PageWindows.tsx +3 -9
- package/src/window/Window.tsx +129 -53
- package/src/window/model.ts +22 -0
- package/src/window/state.tsx +54 -30
- package/test/asset-reference.test.js +53 -0
- package/test/content.test.js +60 -6
- package/test/mutable-assets.test.js +27 -0
- package/test/verify-policy.test.js +92 -0
- package/tsconfig.node.tsbuildinfo +1 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/xp.css/README.md +10 -0
|
@@ -1,37 +1,35 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
|
-
* Refresh src/content/soundcloud.json from SoundCloud's public
|
|
3
|
+
* Refresh src/content/soundcloud.json from SoundCloud's public api-v2 API
|
|
4
4
|
* (no SoundCloud login required).
|
|
5
5
|
*
|
|
6
6
|
* Strategy
|
|
7
7
|
* --------
|
|
8
|
-
* 1.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* (https://soundcloud.com/oembed?url=...&format=json) to obtain a fresh
|
|
18
|
-
* title. oembed works without a SoundCloud `client_id` (no 401s).
|
|
19
|
-
* 4. The merged set is sorted, deduplicated, and written back to
|
|
20
|
-
* src/content/soundcloud.json in the same shape that the music page
|
|
21
|
-
* verify-policy.mjs expect.
|
|
8
|
+
* 1. Fetch the owner's profile HTML and scrape a fresh `client_id` out of
|
|
9
|
+
* one of the `a-v2.sndcdn.com/assets/*.js` bundles it links. SoundCloud
|
|
10
|
+
* rotates these periodically, so they are never hard-coded.
|
|
11
|
+
* 2. Resolve each owner handle to a numeric user id via
|
|
12
|
+
* `api-v2.soundcloud.com/resolve`, then page through
|
|
13
|
+
* `users/<id>/tracks?representation=full` to get the COMPLETE discography
|
|
14
|
+
* (not just the ~10 tracks the profile page server-renders).
|
|
15
|
+
* 3. Emit the tracks in API order (newest first) in the same JSON shape the
|
|
16
|
+
* music page and verify-policy.mjs expect.
|
|
22
17
|
*
|
|
23
|
-
* Why this replaces the old
|
|
24
|
-
*
|
|
25
|
-
* The previous
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
18
|
+
* Why this replaces the old pagerts + oembed scraper
|
|
19
|
+
* ---------------------------------------------------
|
|
20
|
+
* The previous strategy only saw the most recent ~10 tracks per profile in
|
|
21
|
+
* the SSR'd HTML. When a track was renamed, its URL slug changed too, so
|
|
22
|
+
* the new slug was invisible to the scraper while the old slug's oembed
|
|
23
|
+
* endpoint started returning 404 — the pruning step then deleted the track
|
|
24
|
+
* from the snapshot. The api-v2 discography endpoint returns every track
|
|
25
|
+
* with its current permalink and title, so renames are picked up
|
|
26
|
+
* automatically and no pruning is needed at all.
|
|
27
|
+
*
|
|
28
|
+
* The old script's claim that api-v2 was unusable (HTTP 401) is outdated:
|
|
29
|
+
* the 401s came from stale hard-coded client_ids, not from the endpoint
|
|
30
|
+
* itself. Scraping a fresh one per run sidesteps rotation entirely.
|
|
30
31
|
*/
|
|
31
|
-
import {
|
|
32
|
-
import { readFileSync } from "node:fs";
|
|
33
|
-
import { spawnSync } from "node:child_process";
|
|
34
|
-
import { createRequire } from "node:module";
|
|
32
|
+
import { writeFile } from "node:fs/promises";
|
|
35
33
|
import { dirname, resolve } from "node:path";
|
|
36
34
|
import { fileURLToPath } from "node:url";
|
|
37
35
|
|
|
@@ -43,54 +41,15 @@ const outputFile = resolve(projectRoot, "src/content/soundcloud.json");
|
|
|
43
41
|
/** SoundCloud usernames whose discographies should be published. */
|
|
44
42
|
const SOUND_CLOUD_OWNERS = ["akinevz", "akinevz2"];
|
|
45
43
|
|
|
46
|
-
/** User-Agent sent
|
|
44
|
+
/** User-Agent sent for every outbound HTTP request. */
|
|
47
45
|
const USER_AGENT =
|
|
48
46
|
"Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0";
|
|
49
47
|
|
|
50
|
-
/**
|
|
51
|
-
const
|
|
52
|
-
|
|
53
|
-
/** Fetch timeout (ms) for the oembed calls. */
|
|
54
|
-
const OEMBED_TIMEOUT_MS = 8_000;
|
|
55
|
-
|
|
56
|
-
/** System SoundCloud paths that are not tracks. */
|
|
57
|
-
const SYSTEM_PATHS = new Set(["likes", "sets", "tracks", "comments", "followers", "following"]);
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* SoundCloud paths that look like tracks but are actually profile-level
|
|
61
|
-
* landing pages (e.g. /akinevz/recent, /akinevz/popular). Empty for now
|
|
62
|
-
* but kept as a hook for future filtering.
|
|
63
|
-
*/
|
|
64
|
-
const RESERVED_PATHS = new Set([]);
|
|
65
|
-
|
|
66
|
-
/**
|
|
67
|
-
* Timeout (ms) for the pagerts fetch subprocess. Without this a stalled
|
|
68
|
-
* npx/pagerts process would hang the whole build forever.
|
|
69
|
-
*/
|
|
70
|
-
const PAGERTS_TIMEOUT_MS = 120_000;
|
|
48
|
+
/** Fetch timeout (ms) for every HTTP request. */
|
|
49
|
+
const FETCH_TIMEOUT_MS = 15_000;
|
|
71
50
|
|
|
72
|
-
/**
|
|
73
|
-
|
|
74
|
-
* dependency of this package (node_modules), then falls back to whatever
|
|
75
|
-
* `pagerts` resolves on PATH (e.g. a global install).
|
|
76
|
-
*/
|
|
77
|
-
const resolvePagertsBin = () => {
|
|
78
|
-
try {
|
|
79
|
-
const requireFromProject = createRequire(resolve(projectRoot, "package.json"));
|
|
80
|
-
const pkgJsonPath = requireFromProject.resolve("pagerts/package.json");
|
|
81
|
-
const pkgJson = JSON.parse(readFileSync(pkgJsonPath, "utf8"));
|
|
82
|
-
const bin =
|
|
83
|
-
typeof pkgJson.bin === "string"
|
|
84
|
-
? pkgJson.bin
|
|
85
|
-
: (pkgJson.bin && pkgJson.bin.pagerts);
|
|
86
|
-
if (!bin) throw new Error("pagerts package.json has no bin entry");
|
|
87
|
-
return resolve(dirname(pkgJsonPath), bin);
|
|
88
|
-
} catch {
|
|
89
|
-
// Not installed locally — defer to a global install (or fail with a
|
|
90
|
-
// clear ENOENT from spawnSync).
|
|
91
|
-
return "pagerts";
|
|
92
|
-
}
|
|
93
|
-
};
|
|
51
|
+
/** Retries for transient api-v2 failures (it intermittently returns empty bodies). */
|
|
52
|
+
const API_RETRIES = 3;
|
|
94
53
|
|
|
95
54
|
/**
|
|
96
55
|
* Build the source URL for an owner's profile. SoundCloud recently renamed
|
|
@@ -100,416 +59,195 @@ const resolvePagertsBin = () => {
|
|
|
100
59
|
const getProfileUrl = (owner) => `https://soundcloud.com/${owner}`;
|
|
101
60
|
|
|
102
61
|
/**
|
|
103
|
-
*
|
|
104
|
-
*
|
|
62
|
+
* Fetch a URL with a hard timeout. Returns the Response on any HTTP status
|
|
63
|
+
* (callers decide how to interpret it); throws only on network failure.
|
|
105
64
|
*/
|
|
106
|
-
const
|
|
107
|
-
const
|
|
108
|
-
|
|
109
|
-
"--user-agent",
|
|
110
|
-
USER_AGENT,
|
|
111
|
-
...urls,
|
|
112
|
-
];
|
|
113
|
-
|
|
114
|
-
const result = spawnSync(resolvePagertsBin(), args, {
|
|
115
|
-
cwd: projectRoot,
|
|
116
|
-
encoding: "utf8",
|
|
117
|
-
maxBuffer: 32 * 1024 * 1024,
|
|
118
|
-
timeout: PAGERTS_TIMEOUT_MS,
|
|
119
|
-
});
|
|
120
|
-
|
|
121
|
-
if (result.error) {
|
|
122
|
-
const hint =
|
|
123
|
-
result.error.code === "ENOENT"
|
|
124
|
-
? " — is pagerts installed (npm install) or available globally?"
|
|
125
|
-
: "";
|
|
126
|
-
throw new Error(`pagerts fetch could not run: ${result.error.message}${hint}`);
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
if (result.status !== 0) {
|
|
130
|
-
const stderr = result.stderr?.trim() ?? "";
|
|
131
|
-
const timedOut = result.signal === "SIGTERM";
|
|
132
|
-
throw new Error(
|
|
133
|
-
`pagerts fetch failed (${timedOut ? `timed out after ${PAGERTS_TIMEOUT_MS}ms` : `exit ${result.status}`}): ${stderr || "no stderr"}`,
|
|
134
|
-
);
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
const stdout = result.stdout ?? "";
|
|
138
|
-
const trimmed = stdout.trim();
|
|
139
|
-
if (trimmed.length === 0) {
|
|
140
|
-
throw new Error("pagerts fetch produced empty output");
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
// pagerts writes the JSON array to stdout. Find the first '[' in case any
|
|
144
|
-
// diagnostic lines slipped in.
|
|
145
|
-
const startIndex = trimmed.indexOf("[");
|
|
146
|
-
if (startIndex < 0) {
|
|
147
|
-
throw new Error(`pagerts fetch did not return a JSON array: ${trimmed.slice(0, 200)}`);
|
|
148
|
-
}
|
|
149
|
-
|
|
65
|
+
const fetchWithTimeout = async (url, options = {}) => {
|
|
66
|
+
const controller = new AbortController();
|
|
67
|
+
const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
|
150
68
|
try {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
`Failed to parse pagerts output: ${error instanceof Error ? error.message : String(error)}`,
|
|
159
|
-
);
|
|
160
|
-
}
|
|
161
|
-
};
|
|
162
|
-
|
|
163
|
-
/**
|
|
164
|
-
* Pull track URLs (and their visible SSR titles) out of a single pagerts page
|
|
165
|
-
* descriptor. Only /<owner>/<slug> absolute paths are considered tracks;
|
|
166
|
-
* sub-resources, system links (/likes, /sets, /tracks, /comments), and
|
|
167
|
-
* protocol-relative links are ignored.
|
|
168
|
-
*/
|
|
169
|
-
const extractVisibleTracks = (owner, page) => {
|
|
170
|
-
const resources = Array.isArray(page?.resources) ? page.resources : [];
|
|
171
|
-
const seen = new Map();
|
|
172
|
-
|
|
173
|
-
for (const resource of resources) {
|
|
174
|
-
const text = resource?.text?.value;
|
|
175
|
-
const link = resource?.link?.value;
|
|
176
|
-
if (typeof link !== "string" || link.length === 0) continue;
|
|
177
|
-
if (text === "src" && typeof link === "string" && link.startsWith("/")) {
|
|
178
|
-
// pagerts sometimes surfaces a `text=src` for <a class=...><img src=.../></a>
|
|
179
|
-
// — those still represent real links, so allow them through.
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
// Normalize the link to a path. Skip non-track schemes/hosts.
|
|
183
|
-
let path;
|
|
184
|
-
try {
|
|
185
|
-
const url = new URL(link, getProfileUrl(owner));
|
|
186
|
-
if (url.hostname !== "soundcloud.com" && url.hostname !== "www.soundcloud.com") {
|
|
187
|
-
continue;
|
|
188
|
-
}
|
|
189
|
-
path = url.pathname;
|
|
190
|
-
} catch {
|
|
191
|
-
continue;
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
if (!path.startsWith(`/${owner}/`)) continue;
|
|
195
|
-
const slug = path.slice(`/${owner}/`.length);
|
|
196
|
-
if (slug.length === 0) continue;
|
|
197
|
-
if (SYSTEM_PATHS.has(slug)) continue;
|
|
198
|
-
if (RESERVED_PATHS.has(slug)) continue;
|
|
199
|
-
if (/\/.*\//.test(slug)) continue; // ignore nested paths
|
|
200
|
-
|
|
201
|
-
const title = typeof text === "string" && text.length > 0 ? text.trim() : null;
|
|
202
|
-
if (!seen.has(path)) {
|
|
203
|
-
seen.set(path, title);
|
|
204
|
-
} else if (!seen.get(path) && title) {
|
|
205
|
-
seen.set(path, title);
|
|
206
|
-
}
|
|
69
|
+
return await fetch(url, {
|
|
70
|
+
...options,
|
|
71
|
+
headers: { "User-Agent": USER_AGENT, ...options.headers },
|
|
72
|
+
signal: controller.signal,
|
|
73
|
+
});
|
|
74
|
+
} finally {
|
|
75
|
+
clearTimeout(timer);
|
|
207
76
|
}
|
|
208
|
-
|
|
209
|
-
return seen;
|
|
210
77
|
};
|
|
211
78
|
|
|
212
79
|
/**
|
|
213
|
-
* Fetch
|
|
214
|
-
*
|
|
80
|
+
* Fetch a JSON document from api-v2 with retries. api-v2 intermittently
|
|
81
|
+
* returns an empty body even with a 200, so retry until we get parseable
|
|
82
|
+
* JSON. Throws with the HTTP status on definitive failure.
|
|
215
83
|
*/
|
|
216
|
-
const
|
|
84
|
+
const fetchApiJson = async (url, attempt = 1) => {
|
|
217
85
|
try {
|
|
218
|
-
const
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
if (typeof track?.url !== "string") continue;
|
|
224
|
-
try {
|
|
225
|
-
const url = new URL(track.url);
|
|
226
|
-
if (url.hostname !== "soundcloud.com" && url.hostname !== "www.soundcloud.com") continue;
|
|
227
|
-
const key = `${url.pathname.replace(/^\//, "").split("/")[0]}|${url.pathname}`;
|
|
228
|
-
if (!map.has(key)) {
|
|
229
|
-
map.set(key, typeof track.title === "string" ? track.title : null);
|
|
230
|
-
}
|
|
231
|
-
} catch {
|
|
232
|
-
// ignore malformed entries
|
|
233
|
-
}
|
|
86
|
+
const response = await fetchWithTimeout(url, {
|
|
87
|
+
headers: { Accept: "application/json" },
|
|
88
|
+
});
|
|
89
|
+
if (!response.ok) {
|
|
90
|
+
throw new Error(`HTTP ${response.status} for ${new URL(url).pathname}`);
|
|
234
91
|
}
|
|
235
|
-
return
|
|
92
|
+
return await response.json();
|
|
236
93
|
} catch (error) {
|
|
237
|
-
if (
|
|
238
|
-
|
|
94
|
+
if (attempt < API_RETRIES) {
|
|
95
|
+
await new Promise((resolveSleep) => setTimeout(resolveSleep, 1_000 * attempt));
|
|
96
|
+
return fetchApiJson(url, attempt + 1);
|
|
239
97
|
}
|
|
240
98
|
throw error;
|
|
241
99
|
}
|
|
242
100
|
};
|
|
243
101
|
|
|
244
102
|
/**
|
|
245
|
-
*
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
/**
|
|
250
|
-
* Build an oembed URL for a given SoundCloud track URL.
|
|
251
|
-
*/
|
|
252
|
-
const oembedUrl = (trackUrl) => {
|
|
253
|
-
const url = new URL("https://soundcloud.com/oembed");
|
|
254
|
-
url.searchParams.set("url", trackUrl);
|
|
255
|
-
url.searchParams.set("format", "json");
|
|
256
|
-
return url.toString();
|
|
257
|
-
};
|
|
258
|
-
|
|
259
|
-
/**
|
|
260
|
-
* Fetch an oembed JSON document with a hard timeout. Returns the parsed
|
|
261
|
-
* document on success, or null on any failure.
|
|
103
|
+
* Scrape a fresh api-v2 `client_id` from one of the JS asset bundles linked
|
|
104
|
+
* by the owner's profile HTML. SoundCloud rotates these periodically, so
|
|
105
|
+
* they must never be hard-coded. Throws if no bundle yields one.
|
|
262
106
|
*/
|
|
263
|
-
const
|
|
264
|
-
const
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
} finally {
|
|
280
|
-
clearTimeout(timer);
|
|
107
|
+
const resolveClientId = async () => {
|
|
108
|
+
const profileResponse = await fetchWithTimeout(getProfileUrl(SOUND_CLOUD_OWNERS[0]));
|
|
109
|
+
if (!profileResponse.ok) {
|
|
110
|
+
throw new Error(`Could not fetch profile HTML: HTTP ${profileResponse.status}`);
|
|
111
|
+
}
|
|
112
|
+
const html = await profileResponse.text();
|
|
113
|
+
|
|
114
|
+
const assetUrls = [
|
|
115
|
+
...new Set(
|
|
116
|
+
[...html.matchAll(/https:\/\/a-v2\.sndcdn\.com\/assets\/[a-zA-Z0-9._-]+\.js/g)].map(
|
|
117
|
+
(match) => match[0],
|
|
118
|
+
),
|
|
119
|
+
),
|
|
120
|
+
];
|
|
121
|
+
if (assetUrls.length === 0) {
|
|
122
|
+
throw new Error("No SoundCloud asset bundles found in profile HTML");
|
|
281
123
|
}
|
|
282
|
-
};
|
|
283
|
-
|
|
284
|
-
/**
|
|
285
|
-
* Run `worker` over `items` with at most `concurrency` tasks in flight.
|
|
286
|
-
* Returns an array of results in the same order as `items`.
|
|
287
|
-
*/
|
|
288
|
-
const mapWithConcurrency = async (items, concurrency, worker) => {
|
|
289
|
-
const results = new Array(items.length);
|
|
290
|
-
let cursor = 0;
|
|
291
124
|
|
|
292
|
-
const
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
}
|
|
125
|
+
for (const assetUrl of assetUrls) {
|
|
126
|
+
const assetResponse = await fetchWithTimeout(assetUrl);
|
|
127
|
+
if (!assetResponse.ok) continue;
|
|
128
|
+
const js = await assetResponse.text();
|
|
129
|
+
const match = js.match(/client_id:"([a-zA-Z0-9]{20,40})"/);
|
|
130
|
+
if (match) return match[1];
|
|
131
|
+
}
|
|
299
132
|
|
|
300
|
-
|
|
301
|
-
{ length: Math.max(1, Math.min(concurrency, items.length)) },
|
|
302
|
-
() => next(),
|
|
303
|
-
);
|
|
304
|
-
await Promise.all(workers);
|
|
305
|
-
return results;
|
|
133
|
+
throw new Error("No client_id found in any SoundCloud asset bundle");
|
|
306
134
|
};
|
|
307
135
|
|
|
308
136
|
/**
|
|
309
|
-
*
|
|
310
|
-
* SoundCloud serves a 200 HTML page for deleted tracks, so the track URL
|
|
311
|
-
* itself is useless for liveness detection. The oembed endpoint, however,
|
|
312
|
-
* reliably returns HTTP 404 for deleted/private tracks and 200 for live
|
|
313
|
-
* ones. Returns the HTTP status code, or null on network failure (treated
|
|
314
|
-
* as "keep" — we don't want to trim tracks due to transient errors).
|
|
137
|
+
* Resolve an owner handle to its numeric SoundCloud user id.
|
|
315
138
|
*/
|
|
316
|
-
const
|
|
317
|
-
const
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
signal: controller.signal,
|
|
324
|
-
});
|
|
325
|
-
return response.status;
|
|
326
|
-
} catch {
|
|
327
|
-
return null;
|
|
328
|
-
} finally {
|
|
329
|
-
clearTimeout(timer);
|
|
139
|
+
const resolveUserId = async (owner, clientId) => {
|
|
140
|
+
const resolveUrl = new URL("https://api-v2.soundcloud.com/resolve");
|
|
141
|
+
resolveUrl.searchParams.set("url", getProfileUrl(owner));
|
|
142
|
+
resolveUrl.searchParams.set("client_id", clientId);
|
|
143
|
+
const user = await fetchApiJson(resolveUrl.toString());
|
|
144
|
+
if (typeof user?.id !== "number") {
|
|
145
|
+
throw new Error(`Could not resolve user id for ${owner}`);
|
|
330
146
|
}
|
|
147
|
+
return user.id;
|
|
331
148
|
};
|
|
332
149
|
|
|
333
150
|
/**
|
|
334
|
-
*
|
|
335
|
-
*
|
|
336
|
-
*
|
|
337
|
-
* list and logs what was removed.
|
|
151
|
+
* Page through the complete discography for a user. Returns tracks in API
|
|
152
|
+
* order (newest first). Note: `next_href` omits `client_id`, so it must be
|
|
153
|
+
* re-appended on every page or page 2+ returns HTTP 401.
|
|
338
154
|
*/
|
|
339
|
-
const
|
|
340
|
-
const
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
for (const { track, status } of statuses) {
|
|
350
|
-
if (status === 404) {
|
|
351
|
-
dead.push(track);
|
|
352
|
-
} else {
|
|
353
|
-
alive.push(track);
|
|
155
|
+
const fetchAllTracks = async (owner, userId, clientId) => {
|
|
156
|
+
const tracks = [];
|
|
157
|
+
let href = `https://api-v2.soundcloud.com/users/${userId}/tracks?client_id=${clientId}&limit=200&representation=full`;
|
|
158
|
+
|
|
159
|
+
while (href) {
|
|
160
|
+
const page = await fetchApiJson(href);
|
|
161
|
+
const collection = Array.isArray(page?.collection) ? page.collection : [];
|
|
162
|
+
for (const track of collection) {
|
|
163
|
+
if (typeof track?.permalink_url !== "string") continue;
|
|
164
|
+
tracks.push(track);
|
|
354
165
|
}
|
|
166
|
+
const nextHref = typeof page?.next_href === "string" ? page.next_href : null;
|
|
167
|
+
href = nextHref
|
|
168
|
+
? `${nextHref}${nextHref.includes("?") ? "&" : "?"}client_id=${clientId}`
|
|
169
|
+
: null;
|
|
355
170
|
}
|
|
356
171
|
|
|
357
|
-
if (
|
|
358
|
-
|
|
359
|
-
`Trimmed ${dead.length} dead track(s) (oembed 404):\n` +
|
|
360
|
-
dead.map((t) => ` - ${t.url} (${t.title})`).join("\n") +
|
|
361
|
-
"\n",
|
|
362
|
-
);
|
|
363
|
-
} else {
|
|
364
|
-
process.stdout.write("No dead tracks found.\n");
|
|
172
|
+
if (tracks.length === 0) {
|
|
173
|
+
throw new Error(`Discography for ${owner} came back empty`);
|
|
365
174
|
}
|
|
366
|
-
|
|
367
|
-
return alive;
|
|
175
|
+
return tracks;
|
|
368
176
|
};
|
|
369
177
|
|
|
370
178
|
/**
|
|
371
|
-
*
|
|
179
|
+
* Map an api-v2 track document to the snapshot shape. Falls back to the
|
|
180
|
+
* URL slug if the title is missing.
|
|
372
181
|
*/
|
|
373
|
-
const
|
|
374
|
-
|
|
375
|
-
const
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
const mergeTracks = (visibleByOwner, previous) => {
|
|
386
|
-
const merged = new Map(); // key: owner|path -> track
|
|
387
|
-
const orderedKeys = [];
|
|
388
|
-
|
|
389
|
-
// Pass 1: visible tracks per owner, in pagerts-emitted order.
|
|
390
|
-
for (const owner of SOUND_CLOUD_OWNERS) {
|
|
391
|
-
const visible = visibleByOwner.get(owner);
|
|
392
|
-
if (!visible) continue;
|
|
393
|
-
for (const [path, title] of visible.entries()) {
|
|
394
|
-
const url = `https://soundcloud.com${path}`;
|
|
395
|
-
const key = `${owner}|${path}`;
|
|
396
|
-
if (!merged.has(key)) {
|
|
397
|
-
merged.set(key, { owner, path, title, url });
|
|
398
|
-
orderedKeys.push(key);
|
|
399
|
-
}
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
// Pass 2: append any track URL that was in the previous snapshot but not
|
|
404
|
-
// visible in the current pass. This protects the long tail of the
|
|
405
|
-
// discography from being lost when newer uploads push older ones off
|
|
406
|
-
// SoundCloud's first page.
|
|
407
|
-
for (const [key, title] of previous.entries()) {
|
|
408
|
-
if (merged.has(key)) continue;
|
|
409
|
-
const [owner, path] = key.split("|");
|
|
410
|
-
if (!SOUND_CLOUD_OWNERS.includes(owner)) continue;
|
|
411
|
-
const url = `https://soundcloud.com${path}`;
|
|
412
|
-
merged.set(key, { owner, path, title, url });
|
|
413
|
-
orderedKeys.push(key);
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
return orderedKeys.map((key) => merged.get(key));
|
|
182
|
+
const toSnapshotTrack = (owner, track) => {
|
|
183
|
+
const url = track.permalink_url;
|
|
184
|
+
const path = `/${url.replace(/^https?:\/\/(www\.)?soundcloud\.com\//, "")}`;
|
|
185
|
+
return {
|
|
186
|
+
owner,
|
|
187
|
+
path,
|
|
188
|
+
title:
|
|
189
|
+
typeof track.title === "string" && track.title.length > 0
|
|
190
|
+
? track.title
|
|
191
|
+
: path.split("/").pop(),
|
|
192
|
+
url,
|
|
193
|
+
};
|
|
417
194
|
};
|
|
418
195
|
|
|
419
196
|
/**
|
|
420
|
-
*
|
|
421
|
-
*
|
|
197
|
+
* Fetch the complete discography for one owner. Returns the snapshot-shaped
|
|
198
|
+
* track list in API order (newest first).
|
|
422
199
|
*/
|
|
423
|
-
const
|
|
424
|
-
const
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
return stripAuthorSuffix(oembed.title);
|
|
428
|
-
}
|
|
429
|
-
if (typeof track.title === "string" && track.title.length > 0) {
|
|
430
|
-
return track.title;
|
|
431
|
-
}
|
|
432
|
-
return track.path.split("/").pop() ?? track.path;
|
|
433
|
-
});
|
|
434
|
-
|
|
435
|
-
return tracks.map((track, index) => ({ ...track, title: titles[index] }));
|
|
200
|
+
const fetchOwnerDiscography = async (owner, clientId) => {
|
|
201
|
+
const userId = await resolveUserId(owner, clientId);
|
|
202
|
+
const apiTracks = await fetchAllTracks(owner, userId, clientId);
|
|
203
|
+
return apiTracks.map((track) => toSnapshotTrack(owner, track));
|
|
436
204
|
};
|
|
437
205
|
|
|
438
206
|
export const run = async () => {
|
|
439
|
-
const
|
|
440
|
-
|
|
207
|
+
const clientId = await resolveClientId();
|
|
208
|
+
process.stdout.write(`Resolved SoundCloud client_id: ${clientId}\n`);
|
|
441
209
|
|
|
442
|
-
const
|
|
210
|
+
const discographies = new Map();
|
|
443
211
|
for (const owner of SOUND_CLOUD_OWNERS) {
|
|
444
|
-
|
|
445
|
-
if (typeof entry?.url !== "string") return false;
|
|
446
|
-
try {
|
|
447
|
-
return new URL(entry.url).pathname === `/${owner}`;
|
|
448
|
-
} catch {
|
|
449
|
-
return false;
|
|
450
|
-
}
|
|
451
|
-
});
|
|
452
|
-
if (!page) {
|
|
453
|
-
process.stderr.write(`pagerts returned no page for ${owner}\n`);
|
|
454
|
-
visibleByOwner.set(owner, new Map());
|
|
455
|
-
continue;
|
|
456
|
-
}
|
|
457
|
-
visibleByOwner.set(owner, extractVisibleTracks(owner, page));
|
|
212
|
+
discographies.set(owner, await fetchOwnerDiscography(owner, clientId));
|
|
458
213
|
}
|
|
459
214
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
const
|
|
464
|
-
|
|
465
|
-
const asOfUploadingTrackCount = enriched.length;
|
|
466
|
-
const source = getProfileUrl(SOUND_CLOUD_OWNERS[0]);
|
|
467
|
-
|
|
468
|
-
// Build per-owner track lists in the order: visible (newest first),
|
|
469
|
-
// then any long-tail entries from the previous snapshot. The component
|
|
470
|
-
// groups the discography by `owner`, so emitting a `profiles` array
|
|
471
|
-
// ensures every owner gets its own section — not just the first one.
|
|
472
|
-
const tracksByOwner = new Map();
|
|
473
|
-
for (const owner of SOUND_CLOUD_OWNERS) tracksByOwner.set(owner, []);
|
|
474
|
-
for (const track of enriched) {
|
|
475
|
-
const bucket = tracksByOwner.get(track.owner);
|
|
476
|
-
if (bucket) bucket.push(track);
|
|
477
|
-
}
|
|
215
|
+
// Flat track list in per-owner API order (newest first). The music page
|
|
216
|
+
// filters this array by `owner`, so per-owner relative order is what the
|
|
217
|
+
// UI displays.
|
|
218
|
+
const tracks = SOUND_CLOUD_OWNERS.flatMap((owner) => discographies.get(owner) ?? []);
|
|
219
|
+
const trackCount = tracks.length;
|
|
478
220
|
|
|
221
|
+
// Per-owner sections. The component groups the discography by `owner`,
|
|
222
|
+
// so emitting a `profiles` array ensures every owner gets its own
|
|
223
|
+
// section — not just the first one.
|
|
479
224
|
const profiles = SOUND_CLOUD_OWNERS.map((owner) => {
|
|
480
|
-
const ownerTracks =
|
|
225
|
+
const ownerTracks = discographies.get(owner) ?? [];
|
|
481
226
|
return {
|
|
482
227
|
owner,
|
|
483
228
|
source: getProfileUrl(owner),
|
|
484
229
|
profileImageUrl: null,
|
|
485
230
|
trackCount: ownerTracks.length,
|
|
486
|
-
tracks: ownerTracks
|
|
487
|
-
owner: track.owner,
|
|
488
|
-
path: track.path,
|
|
489
|
-
title: track.title,
|
|
490
|
-
url: track.url,
|
|
491
|
-
})),
|
|
231
|
+
tracks: ownerTracks,
|
|
492
232
|
};
|
|
493
233
|
});
|
|
494
234
|
|
|
495
235
|
const result = {
|
|
496
|
-
source,
|
|
236
|
+
source: getProfileUrl(SOUND_CLOUD_OWNERS[0]),
|
|
497
237
|
generatedAt: new Date().toISOString(),
|
|
498
|
-
asOfUploadingTrackCount,
|
|
499
|
-
trackCount
|
|
500
|
-
tracks
|
|
501
|
-
owner: track.owner,
|
|
502
|
-
path: track.path,
|
|
503
|
-
title: track.title,
|
|
504
|
-
url: track.url,
|
|
505
|
-
})),
|
|
238
|
+
asOfUploadingTrackCount: trackCount,
|
|
239
|
+
trackCount,
|
|
240
|
+
tracks,
|
|
506
241
|
profiles,
|
|
507
242
|
};
|
|
508
243
|
|
|
509
244
|
await writeFile(outputFile, `${JSON.stringify(result, null, 2)}\n`, "utf8");
|
|
510
245
|
|
|
511
246
|
process.stdout.write(
|
|
512
|
-
`Refreshed ${result.tracks.length} SoundCloud track(s) into ${outputFile}\n
|
|
247
|
+
`Refreshed ${result.tracks.length} SoundCloud track(s) into ${outputFile}\n` +
|
|
248
|
+
profiles
|
|
249
|
+
.map((profile) => ` @${profile.owner}: ${profile.trackCount} track(s)\n`)
|
|
250
|
+
.join(""),
|
|
513
251
|
);
|
|
514
252
|
|
|
515
253
|
return result;
|