spotifify 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +2 -2
- package/README.zh-CN.md +2 -2
- package/package.json +1 -1
- package/src/cli.ts +40 -5
- package/src/match/matcher.ts +24 -5
- package/src/sources/local/source.ts +14 -5
- package/src/sources/netease/client.ts +15 -15
- package/src/sources/netease/source.ts +8 -1
- package/src/spotify/api.ts +16 -11
- package/src/spotify/auth.ts +5 -9
- package/src/spotify/client.ts +4 -2
- package/src/spotify/localIndex.ts +143 -0
- package/src/state/repo.ts +27 -0
- package/src/sync/apply.ts +37 -10
- package/src/sync/export.ts +31 -10
- package/src/sync/plan.ts +32 -12
- package/src/sync/run.ts +118 -38
- package/src/util/fs.ts +4 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,28 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.1.3](https://github.com/FYWinds/Spotifify/compare/v0.1.2...v0.1.3) (2026-09-05)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
* **match:** let a fingerprint hit decide for a pooled track ([75c2717](https://github.com/FYWinds/Spotifify/commit/75c27178f16b277a5062ff3aff5c396bbc36ec4b))
|
|
9
|
+
* **sources:** unreadable files and partial pulls are not deletions ([0fb80a7](https://github.com/FYWinds/Spotifify/commit/0fb80a7a3af3a03d06543e118be1cb7171fd5880))
|
|
10
|
+
* **spotify:** make playlist writes safe to interrupt and retry ([5882336](https://github.com/FYWinds/Spotifify/commit/5882336ec47f4c8ededef8712312864f7d1d7f16))
|
|
11
|
+
* **sync:** never prune beyond what the run reconciles ([fd1d732](https://github.com/FYWinds/Spotifify/commit/fd1d732fa06cfcff5b855f60ff90c68414d41bb9))
|
|
12
|
+
|
|
13
|
+
## [0.1.2](https://github.com/FYWinds/Spotifify/compare/v0.1.1...v0.1.2) (2026-09-05)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
### Features
|
|
17
|
+
|
|
18
|
+
* **doctor:** check the client's local-file index ([bdc099b](https://github.com/FYWinds/Spotifify/commit/bdc099bfa54abaae2a9f75b4d804fd9fd41f90eb))
|
|
19
|
+
* **sync:** prune superseded exports with --prune ([4db0559](https://github.com/FYWinds/Spotifify/commit/4db05590f9c318e57703a5f8cdcc7be96850a58e))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
### Bug Fixes
|
|
23
|
+
|
|
24
|
+
* **sync:** prune against the listing snapshot ([8c1634a](https://github.com/FYWinds/Spotifify/commit/8c1634a91b2228c4ff4916e42e621ddc54c91737))
|
|
25
|
+
|
|
3
26
|
## [0.1.1](https://github.com/FYWinds/Spotifify/compare/v0.1.0...v0.1.1) (2026-09-05)
|
|
4
27
|
|
|
5
28
|
|
package/README.md
CHANGED
|
@@ -65,9 +65,9 @@ State lives in `~/.spotifify` (`config.toml`, `state.db`, logs); override with `
|
|
|
65
65
|
| Command | What it does |
|
|
66
66
|
|---|---|
|
|
67
67
|
| `init [--force\|--upgrade]` | Write the config template; `--upgrade` merges options added in newer versions into your file (values kept, `.bak` written). |
|
|
68
|
-
| `doctor` | Check config, state db, `ffmpeg`/`fpcalc`, token scopes, search-quota deadline. |
|
|
68
|
+
| `doctor` | Check config, state db, `ffmpeg`/`fpcalc`, token scopes, search-quota deadline, and the desktop client's local-files index (exports it never indexed or indexed with another duration — the two causes of grey rows). |
|
|
69
69
|
| `auth spotify` / `auth netease [--cookie …]` | Log in. |
|
|
70
|
-
| `sync [--dry-run] [--prune] [--source netease\|local] [--playlist NAME] [--skip-match]` | Pull → match → export → plan → apply → report. Exit code `3` = re-authenticate. |
|
|
70
|
+
| `sync [--dry-run] [--prune] [--source netease\|local] [--playlist NAME] [--skip-match]` | Pull → match → export → plan → apply → report. `--prune` also removes superseded local entries and exported files no longer needed — never beyond what the run can account for: with `--playlist`/`--source` nothing is unliked that another mirrored playlist wants, exports still referenced from a playlist outside the run are kept, and a run that mirrors no playlist at all prunes nothing. Exit code `3` = re-authenticate. |
|
|
71
71
|
| `review` | Ink TUI: `j/k` move, `1-9`/`Enter` pick a candidate, `/` custom search, `p` paste a Spotify URL/URI, `o`/`O` open candidate/source in the browser, `l` keep as local file, `s` skip, `u` undo, `?` help. |
|
|
72
72
|
| `status` | Match counts, playlist mappings, last run. |
|
|
73
73
|
| `unmatched [--status local\|review\|all] [--tsv]` | Tracks without a Spotify match and the local file that backs them. |
|
package/README.zh-CN.md
CHANGED
|
@@ -65,9 +65,9 @@ spotifify review # 处理低置信度匹配
|
|
|
65
65
|
| 命令 | 作用 |
|
|
66
66
|
|---|---|
|
|
67
67
|
| `init [--force\|--upgrade]` | 写配置模板;`--upgrade` 把新版本新增的选项合并进现有文件(保留原值,写 `.bak`)。 |
|
|
68
|
-
| `doctor` | 检查配置、状态库、`ffmpeg`/`fpcalc`、token scope
|
|
68
|
+
| `doctor` | 检查配置、状态库、`ffmpeg`/`fpcalc`、token scope、搜索配额截止时间,以及桌面端的本地文件索引(哪些导出没被索引、哪些时长和我们算的不一致——歌单里灰掉的两种原因)。 |
|
|
69
69
|
| `auth spotify` / `auth netease [--cookie …]` | 登录。 |
|
|
70
|
-
| `sync [--dry-run] [--prune] [--source netease\|local] [--playlist 名称] [--skip-match]` | 拉取 → 匹配 → 导出 → 计划 → 执行 →
|
|
70
|
+
| `sync [--dry-run] [--prune] [--source netease\|local] [--playlist 名称] [--skip-match]` | 拉取 → 匹配 → 导出 → 计划 → 执行 → 报告。`--prune` 还会删掉被取代的本地条目和不再需要的导出文件——但只在本次能对账的范围内:带 `--playlist`/`--source` 时不会取消其他镜像歌单想要的喜欢,本次计划外的歌单仍引用的导出文件保留,没有任何镜像歌单的运行什么都不删。退出码 `3` = 需要重新登录。 |
|
|
71
71
|
| `review` | Ink TUI:`j/k` 移动,`1-9`/`Enter` 选候选,`/` 自定义搜索,`p` 粘贴 Spotify 链接/URI,`o`/`O` 在浏览器打开候选/来源,`l` 保持为本地文件,`s` 跳过,`u` 撤销,`?` 帮助。 |
|
|
72
72
|
| `status` | 匹配统计、歌单映射、上次运行。 |
|
|
73
73
|
| `unmatched [--status local\|review\|all] [--tsv]` | 没有 Spotify 匹配的歌以及对应的本地文件。 |
|
package/package.json
CHANGED
package/src/cli.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import { existsSync, mkdirSync, statSync } from "node:fs";
|
|
3
|
-
import {
|
|
3
|
+
import { readFile } from "node:fs/promises";
|
|
4
|
+
import { basename, dirname, join } from "node:path";
|
|
4
5
|
import { Command, InvalidArgumentError } from "commander";
|
|
5
6
|
import qrcode from "qrcode-terminal";
|
|
6
7
|
import { CONFIG_FILENAME, CONFIG_TEMPLATE, loadConfig, missingConfigKeys, stateDir, upgradeConfig, withArtistAliases, type Config } from "./config.ts";
|
|
@@ -12,6 +13,7 @@ import { NeteaseAuthError, NeteaseClient } from "./sources/netease/client.ts";
|
|
|
12
13
|
import { SpotifyApi } from "./spotify/api.ts";
|
|
13
14
|
import { AuthExpiredError, loginPkce, type TokenStore } from "./spotify/auth.ts";
|
|
14
15
|
import { SpotifyClient } from "./spotify/client.ts";
|
|
16
|
+
import { compareExports, findLocalFilesIndexes, parseLocalFilesIndex } from "./spotify/localIndex.ts";
|
|
15
17
|
import { SCOPES, type SpotifyTokens } from "./spotify/types.ts";
|
|
16
18
|
import { openDatabase, schemaVersion } from "./state/db.ts";
|
|
17
19
|
import { Repo } from "./state/repo.ts";
|
|
@@ -42,7 +44,7 @@ interface Ctx {
|
|
|
42
44
|
const program = new Command()
|
|
43
45
|
.name("spotifify")
|
|
44
46
|
.description("Sync Netease Cloud Music playlists and a local library to Spotify")
|
|
45
|
-
.version("0.1.
|
|
47
|
+
.version("0.1.3") // x-release-please-version
|
|
46
48
|
.option("--config <path>", "config file (default: <state-dir>/config.toml)")
|
|
47
49
|
.option("--state-dir <dir>", "state directory (default: ~/.spotifify or $SPOTIFIFY_STATE_DIR)")
|
|
48
50
|
.option("--log-file <path>", "append log lines to this file")
|
|
@@ -91,6 +93,38 @@ function fail(e: unknown): never {
|
|
|
91
93
|
process.exit(EXIT_ERROR);
|
|
92
94
|
}
|
|
93
95
|
|
|
96
|
+
/**
|
|
97
|
+
* Compare `local_export` with the desktop client's own local-files index. Every grey "can't play"
|
|
98
|
+
* row traces back to one of: the client never indexed the file, or it indexed it with another
|
|
99
|
+
* identity (a different duration); both are visible here without touching the network.
|
|
100
|
+
*/
|
|
101
|
+
async function checkClientIndex(repo: Repo, report: (ok: boolean, label: string, detail: string) => void): Promise<void> {
|
|
102
|
+
const exports = repo.listExports();
|
|
103
|
+
if (exports.length === 0) return;
|
|
104
|
+
const indexes = findLocalFilesIndexes();
|
|
105
|
+
if (indexes.length === 0) {
|
|
106
|
+
report(true, "client index", "desktop client index not found; skipped");
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
for (const file of indexes) {
|
|
110
|
+
const written = statSync(file).mtime.toLocaleString();
|
|
111
|
+
const entries = parseLocalFilesIndex(await readFile(file));
|
|
112
|
+
const user = basename(dirname(file)).replace(/-user$/, "");
|
|
113
|
+
if (entries.length === 0) {
|
|
114
|
+
report(false, "client index", `empty for user ${user} (written ${written}); restart the desktop client or toggle the folder under Settings → Local Files`);
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
const c = compareExports(entries, exports);
|
|
118
|
+
const examples = (xs: string[]) => xs.slice(0, 3).join(", ") + (xs.length > 3 ? ", …" : "");
|
|
119
|
+
if (c.mismatched.length > 0) {
|
|
120
|
+
report(false, "client index", `${c.mismatched.length} export(s) indexed with another duration: ${examples(c.mismatched.map((m) => `${m.file} (client ${m.client}s, ours ${m.ours}s)`))}`);
|
|
121
|
+
}
|
|
122
|
+
if (c.missing.length > 0) {
|
|
123
|
+
report(false, "client index", `${c.missing.length} export(s) not indexed by the desktop client (user ${user}, written ${written}): ${examples(c.missing)}; restart the client or toggle the folder`);
|
|
124
|
+
}
|
|
125
|
+
if (c.mismatched.length === 0 && c.missing.length === 0) report(true, "client index", `${c.matched} export(s) indexed with matching identity (user ${user}, written ${written})`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
94
128
|
// ---- init / doctor ----------------------------------------------------------
|
|
95
129
|
|
|
96
130
|
program
|
|
@@ -185,6 +219,7 @@ program
|
|
|
185
219
|
} else {
|
|
186
220
|
report(true, "auth netease", "disabled");
|
|
187
221
|
}
|
|
222
|
+
await checkClientIndex(repo, report);
|
|
188
223
|
db.close();
|
|
189
224
|
} catch (e) {
|
|
190
225
|
report(false, "state.db", e instanceof Error ? e.message : String(e));
|
|
@@ -242,7 +277,7 @@ program
|
|
|
242
277
|
.command("sync")
|
|
243
278
|
.description("pull sources, match, and apply the diff to Spotify")
|
|
244
279
|
.option("--dry-run", "print the plan without applying")
|
|
245
|
-
.option("--prune", "remove tool-added items that left the source (default: report only)")
|
|
280
|
+
.option("--prune", "remove tool-added items that left the source, superseded local entries, and exported files no longer needed (default: report only)")
|
|
246
281
|
.option("--source <kind>", "only this source: netease | local", (v: string) => {
|
|
247
282
|
if (v !== "netease" && v !== "local") throw new InvalidArgumentError("expected netease or local");
|
|
248
283
|
return v;
|
|
@@ -289,10 +324,10 @@ function printSummary(s: SyncSummary): void {
|
|
|
289
324
|
} else if (s.matched.budgetExhausted) {
|
|
290
325
|
console.log(` search budget for this run used up (matching.max_searches_per_run); rerun later or raise the budget`);
|
|
291
326
|
}
|
|
292
|
-
console.log(`plan: create ${s.plan.creates}, add ${s.plan.adds}, move ${s.plan.moves}, prune ${s.plan.prune}, like ${s.plan.likes}, unlike ${s.plan.unlikes}, export ${s.plan.exports}`);
|
|
327
|
+
console.log(`plan: create ${s.plan.creates}, add ${s.plan.adds}, move ${s.plan.moves}, prune ${s.plan.prune}, like ${s.plan.likes}, unlike ${s.plan.unlikes}, export ${s.plan.exports}, remove export ${s.plan.exportGc}`);
|
|
293
328
|
if (s.apply) {
|
|
294
329
|
console.log(
|
|
295
|
-
`applied: created ${s.apply.created}, added ${s.apply.added}, moved ${s.apply.moved}, replaced ${s.apply.replaced}, pruned ${s.apply.pruned}, liked ${s.apply.liked}, unliked ${s.apply.unliked}, exported ${s.apply.exported}${s.apply.exportErrors ? ` (${s.apply.exportErrors} export errors)` : ""}`,
|
|
330
|
+
`applied: created ${s.apply.created}, added ${s.apply.added}, moved ${s.apply.moved}, replaced ${s.apply.replaced}, pruned ${s.apply.pruned}, liked ${s.apply.liked}, unliked ${s.apply.unliked}, exported ${s.apply.exported}${s.apply.exportErrors ? ` (${s.apply.exportErrors} export errors)` : ""}, removed exports ${s.apply.exportsRemoved}`,
|
|
296
331
|
);
|
|
297
332
|
}
|
|
298
333
|
console.log(`match state: ${Object.entries(s.matchCounts).map(([k, v]) => `${k} ${v}`).join(", ")}`);
|
package/src/match/matcher.ts
CHANGED
|
@@ -17,6 +17,7 @@ const TRACK_REF = /^(?:spotify:track:|https?:\/\/open\.spotify\.com\/(?:intl-[a-
|
|
|
17
17
|
/** Dedupes search results by id (and relinked origin id) and scores each one once. */
|
|
18
18
|
class CandidatePool {
|
|
19
19
|
private readonly seen = new Set<string>();
|
|
20
|
+
private readonly byId = new Map<string, Candidate>();
|
|
20
21
|
private readonly candidates: Candidate[] = [];
|
|
21
22
|
|
|
22
23
|
constructor(
|
|
@@ -37,11 +38,23 @@ class CandidatePool {
|
|
|
37
38
|
if (scored.parts.artist < minArtist) continue;
|
|
38
39
|
const c = toCandidate(t.id, t, scored);
|
|
39
40
|
this.candidates.push(c);
|
|
41
|
+
this.byId.set(t.id, c);
|
|
42
|
+
if (t.linked_from) this.byId.set(t.linked_from.id, c);
|
|
40
43
|
added.push(c);
|
|
41
44
|
}
|
|
42
45
|
return added;
|
|
43
46
|
}
|
|
44
47
|
|
|
48
|
+
/** Pooled candidates for these tracks, whichever query admitted them (an identity hit must count even for a track a text query saw first). */
|
|
49
|
+
known(tracks: SpotifyTrack[]): Candidate[] {
|
|
50
|
+
const out = new Set<Candidate>();
|
|
51
|
+
for (const t of tracks) {
|
|
52
|
+
const c = (t.id !== null ? this.byId.get(t.id) : undefined) ?? (t.linked_from !== undefined ? this.byId.get(t.linked_from.id) : undefined);
|
|
53
|
+
if (c) out.add(c);
|
|
54
|
+
}
|
|
55
|
+
return [...out];
|
|
56
|
+
}
|
|
57
|
+
|
|
45
58
|
sorted(): Candidate[] {
|
|
46
59
|
return [...this.candidates].sort((a, b) => b.score - a.score);
|
|
47
60
|
}
|
|
@@ -63,19 +76,24 @@ function toCandidate(id: string, t: SpotifyTrack, scored: { score: number; parts
|
|
|
63
76
|
|
|
64
77
|
type Hit = { by: "isrc" | "fingerprint"; candidate: Candidate } | { by: "auto" };
|
|
65
78
|
|
|
79
|
+
/** ISRCs for a local file (Chromaprint → AcoustID → MusicBrainz); replaceable so the decision path can be exercised without `fpcalc`. */
|
|
80
|
+
export type IsrcLookup = (path: string, contentHash: string, cfg: Config["matching"], repo: Repo, now: number) => Promise<string[]>;
|
|
81
|
+
|
|
66
82
|
export class Matcher {
|
|
67
83
|
private readonly api: SpotifyApi;
|
|
68
84
|
private readonly repo: Repo;
|
|
69
85
|
private readonly cfg: Config;
|
|
70
86
|
private readonly market: string;
|
|
71
87
|
private readonly search: TrackSearch;
|
|
88
|
+
private readonly isrcLookup: IsrcLookup;
|
|
72
89
|
|
|
73
|
-
constructor(deps: { api: SpotifyApi; repo: Repo; cfg: Config; market: string }) {
|
|
90
|
+
constructor(deps: { api: SpotifyApi; repo: Repo; cfg: Config; market: string; isrcLookup?: IsrcLookup }) {
|
|
74
91
|
this.api = deps.api;
|
|
75
92
|
this.repo = deps.repo;
|
|
76
93
|
this.cfg = deps.cfg;
|
|
77
94
|
this.market = deps.market;
|
|
78
95
|
this.search = new TrackSearch(deps.api, deps.repo, deps.cfg.matching, deps.market);
|
|
96
|
+
this.isrcLookup = deps.isrcLookup ?? isrcsByFingerprint;
|
|
79
97
|
}
|
|
80
98
|
|
|
81
99
|
/** Network searches spent by this matcher (cache hits excluded). */
|
|
@@ -92,10 +110,11 @@ export class Matcher {
|
|
|
92
110
|
for (const q of queries) {
|
|
93
111
|
const tracks = await this.search.search(q, now);
|
|
94
112
|
const added = pool.add(tracks, true, q === bareTitle ? BARE_TITLE_MIN_ARTIST : 0);
|
|
95
|
-
if (added.length === 0) continue;
|
|
96
113
|
if (q.startsWith("isrc:")) {
|
|
97
|
-
|
|
98
|
-
|
|
114
|
+
const hits = pool.known(tracks);
|
|
115
|
+
if (hits.length === 0) continue;
|
|
116
|
+
let best = hits[0]!;
|
|
117
|
+
for (const c of hits) if (c.score > best.score) best = c;
|
|
99
118
|
return { by: isrcBy, candidate: best };
|
|
100
119
|
}
|
|
101
120
|
if (added.some((c) => passesAutoGate(c.score, c.parts, m))) return { by: "auto" };
|
|
@@ -114,7 +133,7 @@ export class Matcher {
|
|
|
114
133
|
const bareTitle = track.artists.length > 0 ? queryTitle(track.title) : null;
|
|
115
134
|
let hit = await this.runQueries(pool, this.search.queriesFor(track), bareTitle, "isrc", now);
|
|
116
135
|
if (hit === null && track.file && m.fingerprint) {
|
|
117
|
-
const isrcs = await
|
|
136
|
+
const isrcs = await this.isrcLookup(track.file.path, track.file.contentHash, m, this.repo, now);
|
|
118
137
|
for (const isrc of isrcs) {
|
|
119
138
|
hit = await this.runQueries(pool, [`isrc:${isrc}`], null, "fingerprint", now);
|
|
120
139
|
if (hit !== null) break;
|
|
@@ -89,17 +89,26 @@ export class LocalSource implements Source {
|
|
|
89
89
|
|
|
90
90
|
private async track(file: ScannedFile): Promise<SourceTrack | null> {
|
|
91
91
|
const cached = this.cache.get(file.path);
|
|
92
|
-
if (cached?.file !== undefined && cached.file.size === file.size && cached.file.mtimeMs === file.mtimeMs)
|
|
93
|
-
const { id: _id, canonicalKey: _key, lastSeenAt: _seen, ...track } = cached;
|
|
94
|
-
return track;
|
|
95
|
-
}
|
|
92
|
+
if (cached?.file !== undefined && cached.file.size === file.size && cached.file.mtimeMs === file.mtimeMs) return stripRow(cached);
|
|
96
93
|
try {
|
|
97
94
|
const contentHash = await hashFile(file.path);
|
|
98
95
|
const meta = await describe(file.path, this.cfg.filename_pattern);
|
|
99
96
|
return { kind: "local", externalId: file.path, ...meta, file: { path: file.path, contentHash, size: file.size, mtimeMs: file.mtimeMs } };
|
|
100
97
|
} catch (e) {
|
|
101
|
-
|
|
98
|
+
// A file that cannot be read right now (half-downloaded, locked, damaged) is not a file that left the
|
|
99
|
+
// library: the previous row stands until it can be read again, so the track is never treated as removed.
|
|
100
|
+
const error = e instanceof Error ? e.message : String(e);
|
|
101
|
+
if (cached?.file !== undefined) {
|
|
102
|
+
log.warn(`cannot read ${file.path}; keeping its previous metadata: ${error}`);
|
|
103
|
+
return stripRow(cached);
|
|
104
|
+
}
|
|
105
|
+
log.warn(`skipping unreadable file ${file.path}: ${error}`);
|
|
102
106
|
return null;
|
|
103
107
|
}
|
|
104
108
|
}
|
|
105
109
|
}
|
|
110
|
+
|
|
111
|
+
function stripRow(row: SourceTrackRow): SourceTrack {
|
|
112
|
+
const { id: _id, canonicalKey: _key, lastSeenAt: _seen, ...track } = row;
|
|
113
|
+
return track;
|
|
114
|
+
}
|
|
@@ -35,26 +35,26 @@ export class NeteaseAuthError extends Error {
|
|
|
35
35
|
const LoginStatusBody = z.object({
|
|
36
36
|
data: z.object({ profile: z.object({ userId: z.number(), nickname: z.string().default("") }).nullish() }).optional(),
|
|
37
37
|
});
|
|
38
|
+
// Fields a pull cannot do without (`playlist`, `trackIds`) are required: a shape change must fail the run, not
|
|
39
|
+
// read as "the user has no playlists / the playlist is empty" and delete everything downstream.
|
|
38
40
|
const UserPlaylistBody = z.object({
|
|
39
41
|
more: z.boolean().default(false),
|
|
40
|
-
playlist: z
|
|
41
|
-
.
|
|
42
|
-
z.
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
)
|
|
53
|
-
.default([]),
|
|
42
|
+
playlist: z.array(
|
|
43
|
+
z.object({
|
|
44
|
+
id: z.number(),
|
|
45
|
+
name: z.string(),
|
|
46
|
+
creator: z.object({ userId: z.number() }).nullish(),
|
|
47
|
+
userId: z.number().optional(),
|
|
48
|
+
specialType: z.number().default(0),
|
|
49
|
+
trackCount: z.number().default(0),
|
|
50
|
+
updateTime: z.number().default(0),
|
|
51
|
+
trackUpdateTime: z.number().optional(),
|
|
52
|
+
}),
|
|
53
|
+
),
|
|
54
54
|
});
|
|
55
55
|
const PlaylistDetailBody = z.object({
|
|
56
56
|
playlist: z.object({
|
|
57
|
-
trackIds: z.array(z.object({ id: z.number() }))
|
|
57
|
+
trackIds: z.array(z.object({ id: z.number() })),
|
|
58
58
|
updateTime: z.number().default(0),
|
|
59
59
|
trackUpdateTime: z.number().optional(),
|
|
60
60
|
}),
|
|
@@ -78,10 +78,17 @@ export class NeteaseSource implements Source {
|
|
|
78
78
|
}
|
|
79
79
|
|
|
80
80
|
const tracks: SourceTrack[] = [];
|
|
81
|
+
const omitted: string[] = [];
|
|
81
82
|
for (const id of ids) {
|
|
82
83
|
const t = known.get(id) ?? fetched.get(id);
|
|
83
84
|
if (t) tracks.push(t);
|
|
84
|
-
else
|
|
85
|
+
else omitted.push(id);
|
|
86
|
+
}
|
|
87
|
+
if (omitted.length > 0) {
|
|
88
|
+
// Songs the detail endpoint left out were never known, so nothing is lost yet; but the playlist must not be
|
|
89
|
+
// recorded as pulled at this version, or the cache would serve the short list until the playlist changes.
|
|
90
|
+
log.warn("netease song_detail omitted tracks; the playlist will be pulled again next run", { playlist: summary.name, omitted: omitted.length, ids: omitted.slice(0, 5) });
|
|
91
|
+
playlist.sourceUpdatedAt = undefined;
|
|
85
92
|
}
|
|
86
93
|
log.info("netease playlist pulled", { name: summary.name, tracks: tracks.length, fetched: fetched.size, cached: known.size });
|
|
87
94
|
playlists.push({ playlist, tracks });
|
package/src/spotify/api.ts
CHANGED
|
@@ -6,8 +6,8 @@ import type { SpotifyPlaylist, SpotifyPlaylistItem, SpotifyTrack } from "./types
|
|
|
6
6
|
// `/me/library?uris=`; the legacy `/tracks` and `/me/tracks/contains` paths answer 403 for newer apps.
|
|
7
7
|
const ITEM_FIELDS = "next,total,items(added_at,is_local,item(id,uri,name,is_local,duration_ms,artists(id,name),album(id,name)))";
|
|
8
8
|
|
|
9
|
-
/** Playlist item batch limit for add/remove
|
|
10
|
-
const ITEMS_BATCH = 100;
|
|
9
|
+
/** Playlist item batch limit for add/remove; also the most a single replace can hold. */
|
|
10
|
+
export const ITEMS_BATCH = 100;
|
|
11
11
|
/** `/me/library` batch limit (probed: 41+ uris → 400 "Too many uris requested"). */
|
|
12
12
|
const LIBRARY_BATCH = 40;
|
|
13
13
|
|
|
@@ -70,6 +70,12 @@ export class SpotifyApi {
|
|
|
70
70
|
return this.client.paginate<SpotifyPlaylistItem>(`/v1/playlists/${id}/items`, { limit: 50, fields: ITEM_FIELDS });
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
+
/** Current snapshot id only (used to bracket an items listing). */
|
|
74
|
+
async getPlaylistSnapshot(id: string): Promise<string> {
|
|
75
|
+
const res = await this.client.request<{ snapshot_id: string }>("GET", `/v1/playlists/${id}`, { query: { fields: "snapshot_id" } });
|
|
76
|
+
return res.snapshot_id;
|
|
77
|
+
}
|
|
78
|
+
|
|
73
79
|
/** ≤100 per request; `position` advances by batch size so the whole run lands contiguously. Returns the last snapshot_id. */
|
|
74
80
|
async addPlaylistItems(id: string, uris: string[], position?: number): Promise<string> {
|
|
75
81
|
let snapshot = "";
|
|
@@ -86,8 +92,9 @@ export class SpotifyApi {
|
|
|
86
92
|
|
|
87
93
|
/**
|
|
88
94
|
* ≤100 per request. Catalog tracks are removed by URI; local files must be removed by position only
|
|
89
|
-
* (`/items` answers 400 "Invalid base62 id" for a `spotify:local:` URI).
|
|
90
|
-
*
|
|
95
|
+
* (`/items` answers 400 "Invalid base62 id" for a `spotify:local:` URI). Every position batch names the
|
|
96
|
+
* snapshot the positions were read from: Spotify resolves them against that snapshot, so batches never
|
|
97
|
+
* see each other's shifts (or a concurrent edit). Positions are still sent highest first.
|
|
91
98
|
*/
|
|
92
99
|
async removePlaylistItems(id: string, items: Array<{ uri: string; positions?: number[] }>, snapshotId: string): Promise<string> {
|
|
93
100
|
let snapshot = snapshotId;
|
|
@@ -95,7 +102,7 @@ export class SpotifyApi {
|
|
|
95
102
|
const uris = items.filter((x) => x.positions === undefined).map((x) => ({ uri: x.uri }));
|
|
96
103
|
for (const batch of chunk(positions, ITEMS_BATCH)) {
|
|
97
104
|
const res = await this.client.request<{ snapshot_id: string }>("DELETE", `/v1/playlists/${id}/items`, {
|
|
98
|
-
body: { positions: batch, snapshot_id:
|
|
105
|
+
body: { positions: batch, snapshot_id: snapshotId },
|
|
99
106
|
});
|
|
100
107
|
snapshot = res.snapshot_id;
|
|
101
108
|
}
|
|
@@ -115,13 +122,11 @@ export class SpotifyApi {
|
|
|
115
122
|
return res.snapshot_id;
|
|
116
123
|
}
|
|
117
124
|
|
|
118
|
-
/**
|
|
125
|
+
/** One atomic PUT (≤100 items: a longer list would need a follow-up POST that can fail half-way and lose the tail). */
|
|
119
126
|
async replacePlaylistItems(id: string, uris: string[]): Promise<string> {
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
const rest = uris.slice(ITEMS_BATCH);
|
|
124
|
-
return rest.length === 0 ? res.snapshot_id : this.addPlaylistItems(id, rest);
|
|
127
|
+
if (uris.length > ITEMS_BATCH) throw new Error(`replacePlaylistItems: ${uris.length} items exceed the single-request limit of ${ITEMS_BATCH}`);
|
|
128
|
+
const res = await this.client.request<{ snapshot_id: string }>("PUT", `/v1/playlists/${id}/items`, { body: { uris } });
|
|
129
|
+
return res.snapshot_id;
|
|
125
130
|
}
|
|
126
131
|
|
|
127
132
|
/** ≤50 per request; result aligns with `ids` order. */
|
package/src/spotify/auth.ts
CHANGED
|
@@ -48,7 +48,8 @@ async function postToken(form: Record<string, string>, previous?: SpotifyTokens)
|
|
|
48
48
|
access_token: body.access_token,
|
|
49
49
|
refresh_token: refresh,
|
|
50
50
|
expires_at: Date.now() + body.expires_in * 1000 - EXPIRY_MARGIN_MS,
|
|
51
|
-
|
|
51
|
+
// A refresh response may omit `scope` (or an older store may hold ""): the grant did not shrink, so keep what was known.
|
|
52
|
+
scope: body.scope || previous?.scope || SCOPES.join(" "),
|
|
52
53
|
};
|
|
53
54
|
}
|
|
54
55
|
|
|
@@ -110,12 +111,7 @@ export async function loginPkce(opts: { clientId: string; port: number; store: T
|
|
|
110
111
|
}
|
|
111
112
|
}
|
|
112
113
|
|
|
113
|
-
/** PKCE refresh responses may omit `refresh_token`; the previous
|
|
114
|
-
export async function refreshTokens(clientId: string,
|
|
115
|
-
return postToken({ client_id: clientId, grant_type: "refresh_token", refresh_token:
|
|
116
|
-
access_token: "",
|
|
117
|
-
refresh_token: refreshToken,
|
|
118
|
-
expires_at: 0,
|
|
119
|
-
scope: "",
|
|
120
|
-
});
|
|
114
|
+
/** PKCE refresh responses may omit `refresh_token` and `scope`; the previous values are kept in that case. */
|
|
115
|
+
export async function refreshTokens(clientId: string, previous: SpotifyTokens): Promise<SpotifyTokens> {
|
|
116
|
+
return postToken({ client_id: clientId, grant_type: "refresh_token", refresh_token: previous.refresh_token }, previous);
|
|
121
117
|
}
|
package/src/spotify/client.ts
CHANGED
|
@@ -77,7 +77,9 @@ export class SpotifyClient {
|
|
|
77
77
|
log.warn(`Spotify rate limited; waiting ${secs}s`, { method, path: url.pathname });
|
|
78
78
|
throw new RetryableError(`429 on ${method} ${url.pathname}`, waitMs);
|
|
79
79
|
}
|
|
80
|
-
|
|
80
|
+
// A 5xx on a write may arrive after Spotify applied it (a re-sent POST would add twice, a re-sent
|
|
81
|
+
// reorder move twice), so only reads are retried; a failed write ends the run and the next one re-plans.
|
|
82
|
+
if (res.status >= 500 && method === "GET") throw new RetryableError(`${res.status} on ${method} ${url.pathname}`);
|
|
81
83
|
throw new SpotifyHttpError(res.status, text, method, url.pathname);
|
|
82
84
|
},
|
|
83
85
|
{ attempts: ATTEMPTS },
|
|
@@ -107,7 +109,7 @@ export class SpotifyClient {
|
|
|
107
109
|
if (this.refreshing) return this.refreshing;
|
|
108
110
|
const tokens = this.store.load();
|
|
109
111
|
if (tokens === null) throw new AuthExpiredError();
|
|
110
|
-
this.refreshing = refreshTokens(this.clientId, tokens
|
|
112
|
+
this.refreshing = refreshTokens(this.clientId, tokens)
|
|
111
113
|
.then((fresh) => {
|
|
112
114
|
this.store.save(fresh);
|
|
113
115
|
return fresh;
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reader for the desktop client's local-file index (`local-files.bnk`), the source of truth for which
|
|
3
|
+
* files it knows and which identity (tags + its own duration) it computed for each. `spotifify doctor`
|
|
4
|
+
* compares it with `local_export` so a broken index (half-written file, dropped entry, duration
|
|
5
|
+
* mismatch) is diagnosed instead of showing up as grey playlist rows.
|
|
6
|
+
*
|
|
7
|
+
* The format is undocumented ("SPCO" container, protobuf-like records); this parser only walks the
|
|
8
|
+
* repeating record shape observed in the wild and gives up quietly when it does not match:
|
|
9
|
+
* 09 <len> title 09 <len> artist 09 <len> album 10 <varint seconds> … 2c 01 <len> path … 08 01 78 78 04
|
|
10
|
+
* The file is written when the client flushes (shutdown, rescan), so it may lag the live index.
|
|
11
|
+
*/
|
|
12
|
+
import { existsSync, readdirSync } from "node:fs";
|
|
13
|
+
import { homedir } from "node:os";
|
|
14
|
+
import { basename, join } from "node:path";
|
|
15
|
+
import type { LocalExportRow } from "../state/repo.ts";
|
|
16
|
+
import { parseLocalUri } from "./localUri.ts";
|
|
17
|
+
|
|
18
|
+
export interface LocalIndexEntry {
|
|
19
|
+
title: string;
|
|
20
|
+
artist: string;
|
|
21
|
+
album: string;
|
|
22
|
+
durationSec: number;
|
|
23
|
+
path: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const RECORD_SEPARATOR = [0x08, 0x01, 0x78, 0x78, 0x04];
|
|
27
|
+
|
|
28
|
+
/** Every `Users/<id>-user/local-files.bnk` of every known client install location. */
|
|
29
|
+
export function findLocalFilesIndexes(): string[] {
|
|
30
|
+
const home = homedir();
|
|
31
|
+
const local = process.env.LOCALAPPDATA ?? join(home, "AppData", "Local");
|
|
32
|
+
const roots =
|
|
33
|
+
process.platform === "win32"
|
|
34
|
+
? [join(local, "Packages", "SpotifyAB.SpotifyMusic_zpdnekdrzrea0", "LocalState", "Spotify", "Users"), join(local, "Spotify", "Users")]
|
|
35
|
+
: process.platform === "darwin"
|
|
36
|
+
? [join(home, "Library", "Application Support", "Spotify", "Users")]
|
|
37
|
+
: [join(home, ".config", "spotify", "Users"), join(home, ".var", "app", "com.spotify.Client", "config", "spotify", "Users")];
|
|
38
|
+
const out: string[] = [];
|
|
39
|
+
for (const root of roots) {
|
|
40
|
+
if (!existsSync(root)) continue;
|
|
41
|
+
for (const user of readdirSync(root)) {
|
|
42
|
+
const file = join(root, user, "local-files.bnk");
|
|
43
|
+
if (user.endsWith("-user") && existsSync(file)) out.push(file);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function readVarint(b: Uint8Array, at: number): [value: number, next: number] | null {
|
|
50
|
+
let value = 0;
|
|
51
|
+
let shift = 0;
|
|
52
|
+
for (let i = at; i < b.length && shift <= 35; i++, shift += 7) {
|
|
53
|
+
const c = b[i]!;
|
|
54
|
+
value += (c & 0x7f) * 2 ** shift;
|
|
55
|
+
if (c < 0x80) return [value, i + 1];
|
|
56
|
+
}
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
61
|
+
/** paths occasionally carry stray bytes in the file (seen once in 81 records); identity fields never did */
|
|
62
|
+
const lenientDecoder = new TextDecoder("utf-8");
|
|
63
|
+
|
|
64
|
+
/** `09 <varint len> <utf-8>` */
|
|
65
|
+
function readString(b: Uint8Array, at: number): [value: string, next: number] | null {
|
|
66
|
+
if (b[at] !== 0x09) return null;
|
|
67
|
+
const len = readVarint(b, at + 1);
|
|
68
|
+
if (!len) return null;
|
|
69
|
+
const [n, start] = len;
|
|
70
|
+
if (start + n > b.length) return null;
|
|
71
|
+
try {
|
|
72
|
+
return [decoder.decode(b.subarray(start, start + n)), start + n];
|
|
73
|
+
} catch {
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function indexOfSeq(b: Uint8Array, seq: readonly number[], from: number): number {
|
|
79
|
+
outer: for (let i = from; i + seq.length <= b.length; i++) {
|
|
80
|
+
for (let j = 0; j < seq.length; j++) if (b[i + j] !== seq[j]) continue outer;
|
|
81
|
+
return i;
|
|
82
|
+
}
|
|
83
|
+
return -1;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function parseLocalFilesIndex(b: Uint8Array): LocalIndexEntry[] {
|
|
87
|
+
const out: LocalIndexEntry[] = [];
|
|
88
|
+
let at = indexOfSeq(b, RECORD_SEPARATOR, 0);
|
|
89
|
+
while (at !== -1) {
|
|
90
|
+
const next = indexOfSeq(b, RECORD_SEPARATOR, at + RECORD_SEPARATOR.length);
|
|
91
|
+
const end = next === -1 ? b.length : next;
|
|
92
|
+
const entry = parseRecord(b, at + RECORD_SEPARATOR.length, end);
|
|
93
|
+
if (entry) out.push(entry);
|
|
94
|
+
at = next;
|
|
95
|
+
}
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function parseRecord(b: Uint8Array, at: number, end: number): LocalIndexEntry | null {
|
|
100
|
+
const title = readString(b, at);
|
|
101
|
+
if (!title) return null;
|
|
102
|
+
const artist = readString(b, title[1]);
|
|
103
|
+
if (!artist) return null;
|
|
104
|
+
const album = readString(b, artist[1]);
|
|
105
|
+
if (!album) return null;
|
|
106
|
+
if (b[album[1]] !== 0x10) return null;
|
|
107
|
+
const duration = readVarint(b, album[1] + 1);
|
|
108
|
+
if (!duration) return null;
|
|
109
|
+
// the path follows further down the record: 2c 01 <varint len> <utf-8>
|
|
110
|
+
const p = indexOfSeq(b, [0x2c, 0x01], duration[1]);
|
|
111
|
+
if (p === -1 || p >= end) return null;
|
|
112
|
+
const len = readVarint(b, p + 2);
|
|
113
|
+
if (!len) return null;
|
|
114
|
+
const [n, start] = len;
|
|
115
|
+
if (n === 0 || start + n > end) return null;
|
|
116
|
+
return { title: title[0], artist: artist[0], album: album[0], durationSec: duration[0], path: lenientDecoder.decode(b.subarray(start, start + n)) };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export interface IndexComparison {
|
|
120
|
+
/** exports the client has not indexed at all (file names) */
|
|
121
|
+
missing: string[];
|
|
122
|
+
/** exports the client indexed with a different duration: the pasted uri will never link */
|
|
123
|
+
mismatched: Array<{ file: string; ours: number; client: number }>;
|
|
124
|
+
/** exports found with the identical identity */
|
|
125
|
+
matched: number;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Compare `local_export` identities with the client's index, keyed by tags (the client's own key); paths are not compared. */
|
|
129
|
+
export function compareExports(entries: readonly LocalIndexEntry[], exports: readonly LocalExportRow[]): IndexComparison {
|
|
130
|
+
const key = (title: string, artist: string, album: string) => `${title}\u0000${artist}\u0000${album}`;
|
|
131
|
+
const byIdentity = new Map(entries.map((e) => [key(e.title, e.artist, e.album), e]));
|
|
132
|
+
const out: IndexComparison = { missing: [], mismatched: [], matched: 0 };
|
|
133
|
+
for (const e of exports) {
|
|
134
|
+
const p = parseLocalUri(e.localUri);
|
|
135
|
+
if (!p) continue;
|
|
136
|
+
const file = basename(e.exportPath);
|
|
137
|
+
const hit = byIdentity.get(key(p.title, p.artist, p.album));
|
|
138
|
+
if (!hit) out.missing.push(file);
|
|
139
|
+
else if (hit.durationSec !== p.durationSec) out.mismatched.push({ file, ours: p.durationSec ?? -1, client: hit.durationSec });
|
|
140
|
+
else out.matched++;
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
143
|
+
}
|
package/src/state/repo.ts
CHANGED
|
@@ -386,6 +386,29 @@ export class Repo {
|
|
|
386
386
|
})();
|
|
387
387
|
}
|
|
388
388
|
|
|
389
|
+
/** `spotify:local:` uris recorded as present in a managed playlist → ids of the playlists referencing each. */
|
|
390
|
+
localUriReferences(): Map<string, string[]> {
|
|
391
|
+
const out = new Map<string, string[]>();
|
|
392
|
+
for (const r of this.db.query<{ uri: string; spotify_playlist_id: string }, []>("SELECT uri, spotify_playlist_id FROM managed_item WHERE uri LIKE 'spotify:local:%'").all()) {
|
|
393
|
+
let ids = out.get(r.uri);
|
|
394
|
+
if (!ids) {
|
|
395
|
+
ids = [];
|
|
396
|
+
out.set(r.uri, ids);
|
|
397
|
+
}
|
|
398
|
+
ids.push(r.spotify_playlist_id);
|
|
399
|
+
}
|
|
400
|
+
return out;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/** Record exactly which of our local entries a playlist holds after a sync (rows for entries the user removed by hand go away). */
|
|
404
|
+
replaceManagedLocal(spotifyPlaylistId: string, uris: string[], now: number): void {
|
|
405
|
+
const q = this.db.query("INSERT OR IGNORE INTO managed_item (spotify_playlist_id, uri, added_at) VALUES (?, ?, ?)");
|
|
406
|
+
this.db.transaction(() => {
|
|
407
|
+
this.db.run("DELETE FROM managed_item WHERE spotify_playlist_id = ? AND uri LIKE 'spotify:local:%'", [spotifyPlaylistId]);
|
|
408
|
+
for (const u of uris) q.run(spotifyPlaylistId, u, now);
|
|
409
|
+
})();
|
|
410
|
+
}
|
|
411
|
+
|
|
389
412
|
likedIds(): Set<string> {
|
|
390
413
|
return new Set(this.db.query<{ spotify_id: string }, []>("SELECT spotify_id FROM liked").all().map((r) => r.spotify_id));
|
|
391
414
|
}
|
|
@@ -423,6 +446,10 @@ export class Repo {
|
|
|
423
446
|
);
|
|
424
447
|
}
|
|
425
448
|
|
|
449
|
+
deleteExport(key: string): void {
|
|
450
|
+
this.db.run("DELETE FROM local_export WHERE canonical_key = ?", [key]);
|
|
451
|
+
}
|
|
452
|
+
|
|
426
453
|
// ---- caches -----------------------------------------------------------
|
|
427
454
|
|
|
428
455
|
cacheGet<T>(key: string, now: number, ttlMs: number): T | null {
|
package/src/sync/apply.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* remote operation so a crash mid-way leaves a consistent (re-plannable) state. See DESIGN.md §6.
|
|
4
4
|
*/
|
|
5
5
|
import type { Config } from "../config.ts";
|
|
6
|
-
import type
|
|
6
|
+
import { ITEMS_BATCH, type SpotifyApi } from "../spotify/api.ts";
|
|
7
7
|
import { MANAGED_DESCRIPTION } from "../spotify/types.ts";
|
|
8
8
|
import type { Repo } from "../state/repo.ts";
|
|
9
9
|
import { chunk } from "../util/retry.ts";
|
|
@@ -30,15 +30,24 @@ export interface ApplySummary {
|
|
|
30
30
|
unliked: number;
|
|
31
31
|
exported: number;
|
|
32
32
|
exportErrors: number;
|
|
33
|
+
/** exports garbage-collected after the playlist prune (--prune only) */
|
|
34
|
+
exportsRemoved: number;
|
|
33
35
|
}
|
|
34
36
|
|
|
35
37
|
/** Replace the whole playlist only when it saves real calls: more than this many moves AND more than a third of the list. */
|
|
36
38
|
const REPLACE_MIN_MOVES = 5;
|
|
37
39
|
const REPLACE_MOVE_RATIO = 1 / 3;
|
|
38
40
|
|
|
41
|
+
export class PlaylistDriftError extends Error {
|
|
42
|
+
constructor(name: string) {
|
|
43
|
+
super(`playlist "${name}" changed since it was read; nothing was applied to it — rerun \`spotifify sync\``);
|
|
44
|
+
this.name = "PlaylistDriftError";
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
39
48
|
/** Applies playlist and library changes. Exports run separately (`applyExports`) before planning. */
|
|
40
49
|
export async function applyPlan(plan: Plan, deps: ApplyDeps): Promise<ApplySummary> {
|
|
41
|
-
const s: ApplySummary = { created: 0, renamed: 0, added: 0, pruned: 0, moved: 0, replaced: 0, liked: 0, unliked: 0, exported: 0, exportErrors: 0 };
|
|
50
|
+
const s: ApplySummary = { created: 0, renamed: 0, added: 0, pruned: 0, moved: 0, replaced: 0, liked: 0, unliked: 0, exported: 0, exportErrors: 0, exportsRemoved: 0 };
|
|
42
51
|
|
|
43
52
|
for (const p of plan.playlists) await applyPlaylist(p, deps, s);
|
|
44
53
|
|
|
@@ -91,6 +100,14 @@ async function applyPlaylist(p: PlaylistPlan, deps: ApplyDeps, s: ApplySummary):
|
|
|
91
100
|
}
|
|
92
101
|
if (spotifyId === null) throw new Error(`playlist plan for ${p.sourceName} has neither spotifyId nor create`);
|
|
93
102
|
|
|
103
|
+
// Moves and the replace order describe the playlist as it was listed and carry no snapshot Spotify could
|
|
104
|
+
// validate them against (unlike position removals), so an edit since the listing would reorder the wrong
|
|
105
|
+
// rows: such a playlist is checked before anything is written to it.
|
|
106
|
+
const willPrune = deps.prune && p.prune.length > 0;
|
|
107
|
+
if (p.snapshotId !== null && p.moves.length > 0 && (await api.getPlaylistSnapshot(spotifyId)) !== p.snapshotId) {
|
|
108
|
+
throw new PlaylistDriftError(name);
|
|
109
|
+
}
|
|
110
|
+
|
|
94
111
|
if (p.rename) {
|
|
95
112
|
await api.renamePlaylist(spotifyId, p.rename.to);
|
|
96
113
|
name = p.rename.to;
|
|
@@ -98,8 +115,10 @@ async function applyPlaylist(p: PlaylistPlan, deps: ApplyDeps, s: ApplySummary):
|
|
|
98
115
|
log.info("renamed playlist", p.rename);
|
|
99
116
|
}
|
|
100
117
|
|
|
101
|
-
// Replace-all fast path: only
|
|
102
|
-
|
|
118
|
+
// Replace-all fast path: one atomic PUT, only for lists it can hold whole, with no local items, and when
|
|
119
|
+
// moving would cost noticeably more calls.
|
|
120
|
+
const useReplace =
|
|
121
|
+
p.replaceAllowed && p.targetOrder.length <= ITEMS_BATCH && p.moves.length > REPLACE_MIN_MOVES && p.moves.length > p.targetOrder.length * REPLACE_MOVE_RATIO;
|
|
103
122
|
let snapshot: string | null = null;
|
|
104
123
|
|
|
105
124
|
if (useReplace) {
|
|
@@ -112,21 +131,24 @@ async function applyPlaylist(p: PlaylistPlan, deps: ApplyDeps, s: ApplySummary):
|
|
|
112
131
|
s.replaced++;
|
|
113
132
|
log.info("replaced playlist contents", { name, items: p.targetOrder.length });
|
|
114
133
|
} else {
|
|
115
|
-
for (const uris of chunk(p.adds,
|
|
134
|
+
for (const uris of chunk(p.adds, ITEMS_BATCH)) {
|
|
116
135
|
snapshot = await api.addPlaylistItems(spotifyId, uris);
|
|
117
136
|
repo.addManaged(spotifyId, uris, now);
|
|
118
137
|
s.added += uris.length;
|
|
119
138
|
}
|
|
120
|
-
if (
|
|
121
|
-
|
|
122
|
-
|
|
139
|
+
if (willPrune) {
|
|
140
|
+
// Positions come from the planning-time listing, so they are validated against that snapshot
|
|
141
|
+
// (Spotify checks them against the snapshot given, not the current one). Adds only append.
|
|
142
|
+
const base = p.snapshotId ?? snapshot ?? (await api.getPlaylist(spotifyId))?.snapshot_id ?? null;
|
|
143
|
+
if (base === null) throw new Error(`playlist ${spotifyId} vanished during apply`);
|
|
123
144
|
const items = p.prune.map((x) => (x.uri.startsWith("spotify:local:") ? { uri: x.uri, positions: x.positions } : { uri: x.uri }));
|
|
124
|
-
snapshot = await api.removePlaylistItems(spotifyId, items,
|
|
145
|
+
snapshot = await api.removePlaylistItems(spotifyId, items, base);
|
|
125
146
|
repo.removeManaged(spotifyId, p.prune.map((x) => x.uri));
|
|
126
147
|
s.pruned += p.prune.length;
|
|
127
148
|
}
|
|
128
149
|
if (p.moves.length > 0) {
|
|
129
|
-
|
|
150
|
+
// Moves are computed on the post-add/post-prune order, so they chain from the latest write.
|
|
151
|
+
snapshot ??= p.snapshotId ?? (await api.getPlaylist(spotifyId))?.snapshot_id ?? null;
|
|
130
152
|
if (snapshot === null) throw new Error(`playlist ${spotifyId} vanished during apply`);
|
|
131
153
|
for (const m of p.moves) {
|
|
132
154
|
snapshot = await api.reorderPlaylistItems(spotifyId, m.rangeStart, m.insertBefore, snapshot);
|
|
@@ -134,6 +156,11 @@ async function applyPlaylist(p: PlaylistPlan, deps: ApplyDeps, s: ApplySummary):
|
|
|
134
156
|
}
|
|
135
157
|
}
|
|
136
158
|
}
|
|
159
|
+
// Local entries present after this run reference their export from this playlist; an export is only
|
|
160
|
+
// garbage-collected once no playlist references it any more (see planExportGc). Without --prune the
|
|
161
|
+
// stale ones are still there.
|
|
162
|
+
const localPresent = deps.prune ? p.linked : [...p.linked, ...p.prune.map((x) => x.uri).filter((u) => u.startsWith("spotify:local:"))];
|
|
163
|
+
repo.replaceManagedLocal(spotifyId, localPresent, now);
|
|
137
164
|
|
|
138
165
|
if (p.adds.length > 0 || p.moves.length > 0 || p.prune.length > 0) {
|
|
139
166
|
log.info("synced playlist", { name, added: p.adds.length, pruned: deps.prune ? p.prune.length : 0, moves: p.moves.length, awaiting: p.awaiting.length });
|
package/src/sync/export.ts
CHANGED
|
@@ -8,7 +8,7 @@ import { tmpdir } from "node:os";
|
|
|
8
8
|
import { extname, join } from "node:path";
|
|
9
9
|
import type { Config } from "../config.ts";
|
|
10
10
|
import { decryptNcm } from "../sources/local/ncm.ts";
|
|
11
|
-
import type { SourceTrackRow } from "../state/repo.ts";
|
|
11
|
+
import type { LocalExportRow, Repo, SourceTrackRow } from "../state/repo.ts";
|
|
12
12
|
import { buildLocalUri } from "../spotify/localUri.ts";
|
|
13
13
|
import { probeMp3, probeMp4DurationSec } from "./duration.ts";
|
|
14
14
|
import { log } from "../util/log.ts";
|
|
@@ -93,26 +93,47 @@ export async function exportTrack(plan: ExportPlan, track: SourceTrackRow, cfg:
|
|
|
93
93
|
* that fails with EPERM/EBUSY while the client has it open (playing), which clears within seconds.
|
|
94
94
|
*/
|
|
95
95
|
async function placeExport(partPath: string, exportPath: string): Promise<void> {
|
|
96
|
+
await removeFile(exportPath);
|
|
97
|
+
try {
|
|
98
|
+
await link(partPath, exportPath);
|
|
99
|
+
} catch (e) {
|
|
100
|
+
const code = (e as NodeJS.ErrnoException).code;
|
|
101
|
+
if (code !== "EXDEV" && code !== "EPERM" && code !== "ENOSYS" && code !== "ENOTSUP") throw e;
|
|
102
|
+
log.warn("hard link unavailable, renaming instead; the desktop client will only index the file after a restart", { path: exportPath });
|
|
103
|
+
await rename(partPath, exportPath);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Delete with retries: the desktop client holds an exported file open while it plays it (EPERM/EBUSY on Windows). */
|
|
108
|
+
async function removeFile(path: string): Promise<void> {
|
|
96
109
|
await withRetry(
|
|
97
110
|
async () => {
|
|
98
111
|
try {
|
|
99
|
-
await rm(
|
|
112
|
+
await rm(path, { force: true });
|
|
100
113
|
} catch (e) {
|
|
101
114
|
const code = (e as NodeJS.ErrnoException).code;
|
|
102
|
-
if (code === "EPERM" || code === "EBUSY") throw new RetryableError(`${code}
|
|
115
|
+
if (code === "EPERM" || code === "EBUSY") throw new RetryableError(`${code} removing ${path}`);
|
|
103
116
|
throw e;
|
|
104
117
|
}
|
|
105
118
|
},
|
|
106
119
|
{ attempts: 6, baseMs: 500 },
|
|
107
120
|
);
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Garbage-collect exports (file + record). A file that cannot be deleted keeps its record so the next run retries. Returns the number removed. */
|
|
124
|
+
export async function removeExports(rows: readonly LocalExportRow[], repo: Repo): Promise<number> {
|
|
125
|
+
let removed = 0;
|
|
126
|
+
for (const e of rows) {
|
|
127
|
+
try {
|
|
128
|
+
await removeFile(e.exportPath);
|
|
129
|
+
repo.deleteExport(e.canonicalKey);
|
|
130
|
+
removed++;
|
|
131
|
+
log.info("removed export", { path: e.exportPath });
|
|
132
|
+
} catch (err) {
|
|
133
|
+
log.error("export removal failed", { path: e.exportPath, error: err instanceof Error ? err.message : String(err) });
|
|
134
|
+
}
|
|
115
135
|
}
|
|
136
|
+
return removed;
|
|
116
137
|
}
|
|
117
138
|
|
|
118
139
|
/** null: unparsable, or a VBR mp3 whose client duration is not predictable. */
|
package/src/sync/plan.ts
CHANGED
|
@@ -25,6 +25,8 @@ export interface PlaylistPlan {
|
|
|
25
25
|
sourceName: string;
|
|
26
26
|
/** null when the playlist must be created first */
|
|
27
27
|
spotifyId: string | null;
|
|
28
|
+
/** snapshot the remote listing (and therefore `prune[].positions`) belongs to; null when created */
|
|
29
|
+
snapshotId: string | null;
|
|
28
30
|
create: { name: string } | null;
|
|
29
31
|
rename: { from: string; to: string } | null;
|
|
30
32
|
/** spotify:track URIs to POST, in desired order */
|
|
@@ -35,11 +37,13 @@ export interface PlaylistPlan {
|
|
|
35
37
|
prune: Array<{ uri: string; positions: number[] }>;
|
|
36
38
|
/** remote items not managed by this tool; never removed, kept at the tail */
|
|
37
39
|
foreign: string[];
|
|
40
|
+
/** owned local entries that resolve to a desired export: the user's paste landed, so the export is referenced from this playlist */
|
|
41
|
+
linked: string[];
|
|
38
42
|
/** minimal move sequence to reach desired order (after adds, and prune when enabled) */
|
|
39
43
|
moves: Move[];
|
|
40
44
|
/** full target order (after adds/prune) — used by the replace-all fast path */
|
|
41
45
|
targetOrder: string[];
|
|
42
|
-
/** when true, apply may replace the whole playlist instead of moving (no local items involved) */
|
|
46
|
+
/** when true, apply may replace the whole playlist instead of moving (no local or unresolved items involved) */
|
|
43
47
|
replaceAllowed: boolean;
|
|
44
48
|
}
|
|
45
49
|
|
|
@@ -62,18 +66,22 @@ export interface Plan {
|
|
|
62
66
|
playlists: PlaylistPlan[];
|
|
63
67
|
likes: LikePlan;
|
|
64
68
|
exports: ExportPlan[];
|
|
69
|
+
/** export records (and files) no longer needed; removed only with --prune, after the playlist prune */
|
|
70
|
+
exportGc: LocalExportRow[];
|
|
65
71
|
/** canonical keys needing human review */
|
|
66
72
|
reviewPending: number;
|
|
67
73
|
}
|
|
68
74
|
|
|
69
75
|
export interface RemoteItem {
|
|
76
|
+
/** empty when Spotify returned the entry without an item (unresolvable): it keeps its position but is never touched */
|
|
70
77
|
uri: string;
|
|
71
78
|
isLocal: boolean;
|
|
72
79
|
/**
|
|
73
|
-
* A local entry that names one of our exports
|
|
74
|
-
* (different duration segment, tags from an earlier export).
|
|
80
|
+
* A local entry that names one of our exports — exactly, or with an identity the client will never
|
|
81
|
+
* resolve (different duration segment, tags from an earlier export). Ours to remove with --prune
|
|
82
|
+
* when it is no longer desired (superseded by a Spotify match, gone from the source, wrong identity).
|
|
75
83
|
*/
|
|
76
|
-
|
|
84
|
+
owned: boolean;
|
|
77
85
|
}
|
|
78
86
|
|
|
79
87
|
export interface PlaylistPlanInput {
|
|
@@ -82,6 +90,8 @@ export interface PlaylistPlanInput {
|
|
|
82
90
|
targetName: string;
|
|
83
91
|
/** existing remote playlist (already verified to exist), or null */
|
|
84
92
|
spotify: { id: string; name: string } | null;
|
|
93
|
+
/** snapshot id the `remote` listing was taken at (null when `spotify` is null) */
|
|
94
|
+
snapshotId: string | null;
|
|
85
95
|
/** ordered, deduped by uri */
|
|
86
96
|
desired: DesiredItem[];
|
|
87
97
|
/** current remote order; local uris already canonicalized via `resolveRemoteLocalUri` */
|
|
@@ -96,27 +106,27 @@ const LEGACY_TITLE_SUFFIX = / \(local\)$/;
|
|
|
96
106
|
|
|
97
107
|
export interface ResolvedRemoteLocal {
|
|
98
108
|
uri: string;
|
|
99
|
-
|
|
109
|
+
owned: boolean;
|
|
100
110
|
}
|
|
101
111
|
|
|
102
112
|
/**
|
|
103
113
|
* Map a remote local-file uri onto our export identities. Same artist/album/title/duration as an export
|
|
104
114
|
* → the export's `local_uri`. Same artist/album/title but a different identity (wrong or missing
|
|
105
|
-
* duration segment, legacy title suffix) →
|
|
115
|
+
* duration segment, legacy title suffix) → the verbatim uri (removal by position needs it exactly).
|
|
116
|
+
* Both are `owned`. Anything else is left untouched (a foreign local file).
|
|
106
117
|
*/
|
|
107
118
|
export function resolveRemoteLocalUri(remoteUri: string, exports: readonly LocalExportRow[]): ResolvedRemoteLocal {
|
|
108
119
|
const parts = parseLocalUri(remoteUri);
|
|
109
|
-
if (!parts) return { uri: remoteUri,
|
|
120
|
+
if (!parts) return { uri: remoteUri, owned: false };
|
|
110
121
|
const fold = (s: string) => s.replace(LEGACY_TITLE_SUFFIX, "").trim().toLowerCase();
|
|
111
122
|
for (const e of exports) {
|
|
112
123
|
const p = parseLocalUri(e.localUri);
|
|
113
124
|
if (!p) continue;
|
|
114
125
|
if (fold(p.artist) !== fold(parts.artist) || fold(p.album) !== fold(parts.album) || fold(p.title) !== fold(parts.title)) continue;
|
|
115
126
|
const exact = parts.durationSec === p.durationSec && parts.title === p.title;
|
|
116
|
-
|
|
117
|
-
return exact ? { uri: e.localUri, stale: false } : { uri: remoteUri, stale: true };
|
|
127
|
+
return { uri: exact ? e.localUri : remoteUri, owned: true };
|
|
118
128
|
}
|
|
119
|
-
return { uri: buildLocalUri(parts),
|
|
129
|
+
return { uri: buildLocalUri(parts), owned: false };
|
|
120
130
|
}
|
|
121
131
|
|
|
122
132
|
export function computePlaylistPlan(input: PlaylistPlanInput): PlaylistPlan {
|
|
@@ -128,12 +138,14 @@ export function computePlaylistPlan(input: PlaylistPlanInput): PlaylistPlan {
|
|
|
128
138
|
sourcePlaylistId: input.sourcePlaylistId,
|
|
129
139
|
sourceName: input.sourceName,
|
|
130
140
|
spotifyId: null,
|
|
141
|
+
snapshotId: null,
|
|
131
142
|
create: { name: input.targetName },
|
|
132
143
|
rename: null,
|
|
133
144
|
adds: desired.filter((d) => d.kind === "spotify").map((d) => d.uri),
|
|
134
145
|
awaiting: desired.filter((d) => d.kind === "local"),
|
|
135
146
|
prune: [],
|
|
136
147
|
foreign: [],
|
|
148
|
+
linked: [],
|
|
137
149
|
moves: [],
|
|
138
150
|
targetOrder: desired.filter((d) => d.kind === "spotify").map((d) => d.uri),
|
|
139
151
|
replaceAllowed: false,
|
|
@@ -143,13 +155,19 @@ export function computePlaylistPlan(input: PlaylistPlanInput): PlaylistPlan {
|
|
|
143
155
|
const remoteSet = new Set(remote.map((r) => r.uri));
|
|
144
156
|
const adds = desired.filter((d) => d.kind === "spotify" && !remoteSet.has(d.uri)).map((d) => d.uri);
|
|
145
157
|
const awaiting = desired.filter((d) => d.kind === "local" && !remoteSet.has(d.uri));
|
|
158
|
+
const linked = desired.filter((d) => d.kind === "local" && remoteSet.has(d.uri)).map((d) => d.uri);
|
|
146
159
|
|
|
147
160
|
const prune: PlaylistPlan["prune"] = [];
|
|
148
161
|
const pruneByUri = new Map<string, number[]>();
|
|
149
162
|
const foreign: string[] = [];
|
|
163
|
+
let unresolved = false;
|
|
150
164
|
remote.forEach((r, i) => {
|
|
165
|
+
if (r.uri === "") {
|
|
166
|
+
unresolved = true;
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
151
169
|
if (desiredSet.has(r.uri)) return;
|
|
152
|
-
if (managed.has(r.uri) || r.
|
|
170
|
+
if (managed.has(r.uri) || r.owned) {
|
|
153
171
|
let positions = pruneByUri.get(r.uri);
|
|
154
172
|
if (!positions) {
|
|
155
173
|
positions = [];
|
|
@@ -192,14 +210,16 @@ export function computePlaylistPlan(input: PlaylistPlanInput): PlaylistPlan {
|
|
|
192
210
|
sourcePlaylistId: input.sourcePlaylistId,
|
|
193
211
|
sourceName: input.sourceName,
|
|
194
212
|
spotifyId: input.spotify.id,
|
|
213
|
+
snapshotId: input.snapshotId,
|
|
195
214
|
create: null,
|
|
196
215
|
rename: input.spotify.name === input.targetName ? null : { from: input.spotify.name, to: input.targetName },
|
|
197
216
|
adds,
|
|
198
217
|
awaiting,
|
|
199
218
|
prune,
|
|
200
219
|
foreign,
|
|
220
|
+
linked,
|
|
201
221
|
moves,
|
|
202
222
|
targetOrder,
|
|
203
|
-
replaceAllowed: !anyLocal,
|
|
223
|
+
replaceAllowed: !anyLocal && !unresolved,
|
|
204
224
|
};
|
|
205
225
|
}
|
package/src/sync/run.ts
CHANGED
|
@@ -12,13 +12,14 @@ import { NeteaseSource } from "../sources/netease/source.ts";
|
|
|
12
12
|
import type { SourceKind, SourceTrack } from "../sources/types.ts";
|
|
13
13
|
import type { SpotifyApi } from "../spotify/api.ts";
|
|
14
14
|
import { SpotifyHttpError, SpotifyRateLimitedError } from "../spotify/client.ts";
|
|
15
|
-
import { MANAGED_DESCRIPTION } from "../spotify/types.ts";
|
|
15
|
+
import { MANAGED_DESCRIPTION, type SpotifyPlaylistItem } from "../spotify/types.ts";
|
|
16
16
|
import { parseLocalUri } from "../spotify/localUri.ts";
|
|
17
17
|
import type { LocalExportRow, Repo, SourcePlaylistRow } from "../state/repo.ts";
|
|
18
|
-
import { sanitizeFilename } from "../util/fs.ts";
|
|
18
|
+
import { MAX_FILENAME, sanitizeFilename } from "../util/fs.ts";
|
|
19
19
|
import { log } from "../util/log.ts";
|
|
20
20
|
import { mapLimit } from "../util/retry.ts";
|
|
21
21
|
import { applyExports, applyPlan, type ApplySummary } from "./apply.ts";
|
|
22
|
+
import { removeExports } from "./export.ts";
|
|
22
23
|
import { computePlaylistPlan, resolveRemoteLocalUri, type DesiredItem, type ExportPlan, type Plan, type PlaylistPlan, type RemoteItem } from "./plan.ts";
|
|
23
24
|
|
|
24
25
|
export interface SyncOptions {
|
|
@@ -58,7 +59,7 @@ export interface MatchPhaseSummary {
|
|
|
58
59
|
export interface SyncSummary {
|
|
59
60
|
pulled: Record<SourceKind, { playlists: number; tracks: number }>;
|
|
60
61
|
matched: MatchPhaseSummary;
|
|
61
|
-
plan: { creates: number; adds: number; prune: number; moves: number; likes: number; unlikes: number; exports: number; reviewPending: number };
|
|
62
|
+
plan: { creates: number; adds: number; prune: number; moves: number; likes: number; unlikes: number; exports: number; exportGc: number; reviewPending: number };
|
|
62
63
|
apply: ApplySummary | null;
|
|
63
64
|
awaiting: AwaitingEntry[];
|
|
64
65
|
matchCounts: Record<string, number>;
|
|
@@ -92,6 +93,12 @@ export async function runSync(deps: SyncDeps, opts: SyncOptions): Promise<SyncRe
|
|
|
92
93
|
apply.exported = exported.exported;
|
|
93
94
|
apply.exportErrors = exported.errors;
|
|
94
95
|
}
|
|
96
|
+
// After apply: the playlist entries pointing at these files were pruned above, so the files can go —
|
|
97
|
+
// unless some playlist outside this plan still holds an entry for them.
|
|
98
|
+
if (apply && opts.prune) {
|
|
99
|
+
const referenced = repo.localUriReferences();
|
|
100
|
+
apply.exportsRemoved = await removeExports(plan.exportGc.filter((e) => !referenced.has(e.localUri)), repo);
|
|
101
|
+
}
|
|
95
102
|
const summary: SyncSummary = {
|
|
96
103
|
pulled,
|
|
97
104
|
matched,
|
|
@@ -103,6 +110,7 @@ export async function runSync(deps: SyncDeps, opts: SyncOptions): Promise<SyncRe
|
|
|
103
110
|
likes: plan.likes.add.length,
|
|
104
111
|
unlikes: plan.likes.prune.length,
|
|
105
112
|
exports: plan.exports.length,
|
|
113
|
+
exportGc: plan.exportGc.length,
|
|
106
114
|
reviewPending: plan.reviewPending,
|
|
107
115
|
},
|
|
108
116
|
apply,
|
|
@@ -130,6 +138,27 @@ export function planExportsOnly(repo: Repo, cfg: Config, opts: Pick<SyncOptions,
|
|
|
130
138
|
return planExports(repo, repo.listMatches("local").map((m) => m.canonicalKey).filter((k) => keys.has(k)), repo.listExports(), force);
|
|
131
139
|
}
|
|
132
140
|
|
|
141
|
+
/**
|
|
142
|
+
* Exported files no longer needed: the track left every mirrored playlist, or it has a Spotify match
|
|
143
|
+
* now. Only meaningful for a full run — with `--playlist`/`--source` the other playlists' exports
|
|
144
|
+
* would look unneeded — and never when nothing is mirrored (a config slip must not delete everything).
|
|
145
|
+
* An export whose entry sits in a playlist outside `planned` (retired from mirroring, or filtered out)
|
|
146
|
+
* is kept: only the playlists in this plan prune their entries in the same run.
|
|
147
|
+
*/
|
|
148
|
+
export function planExportGc(repo: Repo, cfg: Config, opts: Pick<SyncOptions, "source" | "playlist">, planned: ReadonlySet<string>): LocalExportRow[] {
|
|
149
|
+
if (opts.source || opts.playlist) return [];
|
|
150
|
+
if (selectedSourcePlaylists(repo, cfg, {}).length === 0) return [];
|
|
151
|
+
const needed = selectedKeys(repo, cfg, {});
|
|
152
|
+
// `review` keeps its export: a candidate that still needs a human must not undo a paste (see desiredItems)
|
|
153
|
+
const keep = new Set([...repo.listMatches("local"), ...repo.listMatches("review")].map((m) => m.canonicalKey));
|
|
154
|
+
const referenced = repo.localUriReferences();
|
|
155
|
+
return repo.listExports().filter((e) => {
|
|
156
|
+
if (needed.has(e.canonicalKey) && keep.has(e.canonicalKey)) return false;
|
|
157
|
+
const refs = referenced.get(e.localUri);
|
|
158
|
+
return refs === undefined || refs.every((id) => planned.has(id));
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
|
|
133
162
|
// ---- pull -------------------------------------------------------------------
|
|
134
163
|
|
|
135
164
|
async function pull(deps: SyncDeps, opts: SyncOptions, now: number): Promise<SyncSummary["pulled"]> {
|
|
@@ -244,45 +273,34 @@ export async function buildPlan(deps: SyncDeps, opts: Pick<SyncOptions, "prune"
|
|
|
244
273
|
const me = await api.me();
|
|
245
274
|
const remotePlaylists = (await api.listMyPlaylists()).filter((p) => p.owner.id === me.id);
|
|
246
275
|
const exports = repo.listExports();
|
|
276
|
+
const exportByKey = new Map(exports.map((e) => [e.canonicalKey, e] as const));
|
|
247
277
|
|
|
248
|
-
const
|
|
278
|
+
const mirrored = selectedSourcePlaylists(repo, cfg, {});
|
|
279
|
+
const selected = new Set(selectedSourcePlaylists(repo, cfg, opts).map((p) => p.id));
|
|
249
280
|
|
|
250
281
|
const playlists: PlaylistPlan[] = [];
|
|
282
|
+
/** ids the playlists this run acts on want liked → saved if not yet */
|
|
251
283
|
const likeDesired = new Set<string>();
|
|
284
|
+
/** ids any mirrored playlist wants liked → never pruned, whatever `--playlist`/`--source` selected */
|
|
285
|
+
const likeNeeded = new Set<string>();
|
|
252
286
|
|
|
253
|
-
for (const sp of
|
|
254
|
-
const
|
|
255
|
-
const
|
|
256
|
-
|
|
257
|
-
const
|
|
258
|
-
const likeThis = sp.kind === "netease" ? cfg.netease.like_matched : cfg.local.like_matched;
|
|
259
|
-
|
|
260
|
-
const desired: DesiredItem[] = [];
|
|
261
|
-
const seen = new Set<string>();
|
|
262
|
-
for (const t of tracks) {
|
|
263
|
-
const m = matches.get(t.canonicalKey);
|
|
264
|
-
if (!m) continue;
|
|
265
|
-
let item: DesiredItem | null = null;
|
|
266
|
-
if (m.status === "matched" && m.spotifyUri && m.spotifyId) {
|
|
267
|
-
item = { uri: m.spotifyUri, kind: "spotify", canonicalKey: t.canonicalKey };
|
|
268
|
-
if (likeThis) likeDesired.add(m.spotifyId);
|
|
269
|
-
} else if (m.status === "local") {
|
|
270
|
-
const e = exportByKey.get(t.canonicalKey);
|
|
271
|
-
if (e) item = { uri: e.localUri, kind: "local", canonicalKey: t.canonicalKey };
|
|
272
|
-
}
|
|
273
|
-
if (item && !seen.has(item.uri)) {
|
|
274
|
-
seen.add(item.uri);
|
|
275
|
-
desired.push(item);
|
|
276
|
-
}
|
|
277
|
-
}
|
|
287
|
+
for (const sp of mirrored) {
|
|
288
|
+
const { desired, likes } = desiredItems(repo, cfg, sp, exportByKey);
|
|
289
|
+
for (const id of likes) likeNeeded.add(id);
|
|
290
|
+
if (!selected.has(sp.id)) continue;
|
|
291
|
+
for (const id of likes) likeDesired.add(id);
|
|
278
292
|
|
|
293
|
+
const targetName = cfg.sync.playlist_prefix + sp.name;
|
|
279
294
|
const remote = await resolveRemotePlaylist(sp, targetName, remotePlaylists, deps);
|
|
280
295
|
let remoteItems: RemoteItem[] = [];
|
|
296
|
+
let snapshotId: string | null = null;
|
|
281
297
|
if (remote) {
|
|
282
|
-
|
|
283
|
-
|
|
298
|
+
const listing = await listPlaylistItemsConsistently(api, remote.id);
|
|
299
|
+
snapshotId = listing.snapshotId;
|
|
300
|
+
remoteItems = listing.items.map((it) => {
|
|
301
|
+
if (!it.item) return { uri: "", isLocal: false, owned: false };
|
|
284
302
|
if (it.is_local || it.item.is_local) return { ...resolveRemoteLocalUri(it.item.uri, exports), isLocal: true };
|
|
285
|
-
return { uri: it.item.uri, isLocal: false,
|
|
303
|
+
return { uri: it.item.uri, isLocal: false, owned: false };
|
|
286
304
|
});
|
|
287
305
|
}
|
|
288
306
|
|
|
@@ -292,6 +310,7 @@ export async function buildPlan(deps: SyncDeps, opts: Pick<SyncOptions, "prune"
|
|
|
292
310
|
sourceName: sp.name,
|
|
293
311
|
targetName,
|
|
294
312
|
spotify: remote,
|
|
313
|
+
snapshotId,
|
|
295
314
|
desired,
|
|
296
315
|
remote: remoteItems,
|
|
297
316
|
managed: remote ? repo.managedUris(remote.id) : new Set<string>(),
|
|
@@ -300,15 +319,53 @@ export async function buildPlan(deps: SyncDeps, opts: Pick<SyncOptions, "prune"
|
|
|
300
319
|
);
|
|
301
320
|
}
|
|
302
321
|
|
|
303
|
-
// Likes: everything desired that is not already saved; prune tool-liked ids no
|
|
322
|
+
// Likes: everything desired that is not already saved; prune tool-liked ids no mirrored playlist wants any
|
|
323
|
+
// more. With nothing mirrored there is nothing to reconcile against, so nothing is pruned.
|
|
304
324
|
const likeIds = [...likeDesired];
|
|
305
325
|
const saved = await savedFlags(api, likeIds);
|
|
306
326
|
const likes = {
|
|
307
327
|
add: likeIds.filter((_, i) => !saved[i]),
|
|
308
|
-
prune: [...repo.likedIds()].filter((id) => !
|
|
328
|
+
prune: mirrored.length === 0 ? [] : [...repo.likedIds()].filter((id) => !likeNeeded.has(id)),
|
|
309
329
|
};
|
|
330
|
+
if (mirrored.length === 0 && (repo.likedIds().size > 0 || exports.length > 0)) {
|
|
331
|
+
log.warn("no playlist is mirrored (check netease.include_playlists / local.mirror_playlist); keeping every tool-liked track and export", {
|
|
332
|
+
liked: repo.likedIds().size,
|
|
333
|
+
exports: exports.length,
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const planned = new Set<string>();
|
|
338
|
+
for (const p of playlists) if (p.spotifyId !== null) planned.add(p.spotifyId);
|
|
339
|
+
return { playlists, likes, exports: exportPlans, exportGc: planExportGc(repo, cfg, opts, planned), reviewPending: repo.countMatches().review };
|
|
340
|
+
}
|
|
310
341
|
|
|
311
|
-
|
|
342
|
+
/** What one source playlist wants on Spotify, in source order and deduped by uri, plus the ids it wants liked. */
|
|
343
|
+
function desiredItems(repo: Repo, cfg: Config, sp: SourcePlaylistRow, exportByKey: ReadonlyMap<string, LocalExportRow>): { desired: DesiredItem[]; likes: string[] } {
|
|
344
|
+
const tracks = repo.playlistTracks(sp.id);
|
|
345
|
+
const matches = repo.matchesForKeys(tracks.map((t) => t.canonicalKey));
|
|
346
|
+
const likeThis = sp.kind === "netease" ? cfg.netease.like_matched : cfg.local.like_matched;
|
|
347
|
+
const desired: DesiredItem[] = [];
|
|
348
|
+
const likes: string[] = [];
|
|
349
|
+
const seen = new Set<string>();
|
|
350
|
+
for (const t of tracks) {
|
|
351
|
+
const m = matches.get(t.canonicalKey);
|
|
352
|
+
if (!m) continue;
|
|
353
|
+
let item: DesiredItem | null = null;
|
|
354
|
+
if (m.status === "matched" && m.spotifyUri && m.spotifyId) {
|
|
355
|
+
item = { uri: m.spotifyUri, kind: "spotify", canonicalKey: t.canonicalKey };
|
|
356
|
+
if (likeThis) likes.push(m.spotifyId);
|
|
357
|
+
} else if (m.status === "local" || m.status === "review") {
|
|
358
|
+
// `review` keeps the export already in place: a candidate that still needs a human (a re-search after
|
|
359
|
+
// retry_unmatched_after_days, say) must not undo the user's paste. Only `local` gets a new export.
|
|
360
|
+
const e = exportByKey.get(t.canonicalKey);
|
|
361
|
+
if (e) item = { uri: e.localUri, kind: "local", canonicalKey: t.canonicalKey };
|
|
362
|
+
}
|
|
363
|
+
if (item && !seen.has(item.uri)) {
|
|
364
|
+
seen.add(item.uri);
|
|
365
|
+
desired.push(item);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
return { desired, likes };
|
|
312
369
|
}
|
|
313
370
|
|
|
314
371
|
/** Which of `ids` are already liked. `/me/tracks/contains` is 403 for some development-mode apps; then list the library instead. */
|
|
@@ -324,6 +381,22 @@ async function savedFlags(api: SpotifyApi, ids: string[]): Promise<boolean[]> {
|
|
|
324
381
|
}
|
|
325
382
|
}
|
|
326
383
|
|
|
384
|
+
/**
|
|
385
|
+
* Items plus the snapshot id they belong to. The listing is paginated and the snapshot endpoint is
|
|
386
|
+
* separate, so the listing is bracketed by two snapshot reads and retried while they differ; the
|
|
387
|
+
* snapshot is what position-based removals are validated against, so a wrong one must never be sent.
|
|
388
|
+
*/
|
|
389
|
+
async function listPlaylistItemsConsistently(api: SpotifyApi, id: string): Promise<{ items: SpotifyPlaylistItem[]; snapshotId: string }> {
|
|
390
|
+
for (let attempt = 1; ; attempt++) {
|
|
391
|
+
const before = await api.getPlaylistSnapshot(id);
|
|
392
|
+
const items = await api.getPlaylistItems(id);
|
|
393
|
+
const after = await api.getPlaylistSnapshot(id);
|
|
394
|
+
if (before === after) return { items, snapshotId: after };
|
|
395
|
+
if (attempt === 3) throw new Error(`playlist ${id} keeps changing while it is being read; retry later`);
|
|
396
|
+
log.warn("playlist changed while listing, retrying", { id, attempt });
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
327
400
|
/**
|
|
328
401
|
* Find the remote playlist for a source playlist: the stored mapping if it still exists, else a
|
|
329
402
|
* remote playlist with the target name carrying our description (adoption after state loss), else null.
|
|
@@ -362,9 +435,14 @@ function planExports(repo: Repo, localKeys: string[], exports: LocalExportRow[],
|
|
|
362
435
|
// An export is current when the source is unchanged and its recorded identity is complete
|
|
363
436
|
// (rows written before the duration segment was known cannot match anything the client indexes).
|
|
364
437
|
if (!force && existing && existing.contentHash === t.file.contentHash && parseLocalUri(existing.localUri)?.durationSec !== null) continue;
|
|
365
|
-
|
|
438
|
+
const raw = `${t.artists.join(", ") || "Unknown Artist"} - ${t.title}`;
|
|
439
|
+
let base = sanitizeFilename(raw);
|
|
366
440
|
if (!existing) {
|
|
367
|
-
|
|
441
|
+
// The suffix is appended after truncation, so a name at the length limit still gets a distinct one.
|
|
442
|
+
for (let n = 2; usedNames.has(base.toLowerCase()); n++) {
|
|
443
|
+
const suffix = ` (${n})`;
|
|
444
|
+
base = sanitizeFilename(raw, MAX_FILENAME - suffix.length) + suffix;
|
|
445
|
+
}
|
|
368
446
|
} else {
|
|
369
447
|
base = existing.exportPath.replace(/\.[^.\\/]+$/, "").replace(/^.*[\\/]/, "");
|
|
370
448
|
}
|
|
@@ -396,9 +474,11 @@ export function formatPlan(plan: Plan, prune: boolean): string {
|
|
|
396
474
|
for (const x of p.prune) lines.push(` ${prune ? "-" : "?"} ${x.uri}`);
|
|
397
475
|
}
|
|
398
476
|
lines.push(`likes: +${plan.likes.add.length}, prune ${plan.likes.prune.length}${prune ? "" : " (report only)"}`);
|
|
399
|
-
lines.push(`exports: ${plan.exports.length}`);
|
|
477
|
+
lines.push(`exports: ${plan.exports.length}, remove ${plan.exportGc.length}${prune ? "" : " (report only)"}`);
|
|
400
478
|
for (const e of plan.exports.slice(0, 20)) lines.push(` → ${e.baseName} (${e.sourcePath})`);
|
|
401
479
|
if (plan.exports.length > 20) lines.push(` → … ${plan.exports.length - 20} more`);
|
|
480
|
+
for (const e of plan.exportGc.slice(0, 20)) lines.push(` ${prune ? "-" : "?"} ${e.exportPath}`);
|
|
481
|
+
if (plan.exportGc.length > 20) lines.push(` ${prune ? "-" : "?"} … ${plan.exportGc.length - 20} more`);
|
|
402
482
|
lines.push(`review pending: ${plan.reviewPending}`);
|
|
403
483
|
return lines.join("\n");
|
|
404
484
|
}
|
package/src/util/fs.ts
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { createReadStream } from "node:fs";
|
|
2
2
|
|
|
3
|
+
/** Longest base name (without extension) an export file gets, in UTF-16 units (NTFS's limit is 255). */
|
|
4
|
+
export const MAX_FILENAME = 150;
|
|
5
|
+
|
|
3
6
|
/** Replace characters Windows/NTFS rejects and trim trailing dots/spaces. */
|
|
4
|
-
export function sanitizeFilename(name: string, max =
|
|
7
|
+
export function sanitizeFilename(name: string, max = MAX_FILENAME): string {
|
|
5
8
|
const cleaned = name
|
|
6
9
|
.replace(/[<>:"/\\|?*\u0000-\u001f]/g, "_")
|
|
7
10
|
.replace(/\s+/g, " ")
|