podcast-dl 11.6.2 → 11.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/archive.js +18 -6
- package/bin/async.js +16 -19
- package/bin/bin.js +6 -4
- package/bin/items.js +12 -8
- package/bin/meta.js +12 -4
- package/bin/util.js +4 -9
- package/package.json +1 -1
package/bin/archive.js
CHANGED
|
@@ -7,25 +7,37 @@ export const getArchiveKey = ({ prefix, name }) => {
|
|
|
7
7
|
return `${prefix}-${name}`;
|
|
8
8
|
};
|
|
9
9
|
|
|
10
|
+
export const getArchiveKeys = ({ prefix, name, guid }) => {
|
|
11
|
+
const legacyKey = name ? getArchiveKey({ prefix, name }) : null;
|
|
12
|
+
const guidKey = guid ? getArchiveKey({ prefix, name: guid }) : null;
|
|
13
|
+
return [legacyKey, guidKey].filter(Boolean);
|
|
14
|
+
};
|
|
15
|
+
|
|
10
16
|
export const getArchive = (archive) => {
|
|
11
17
|
const archiveContent = getJsonFile(archive);
|
|
12
18
|
return archiveContent === null ? [] : archiveContent;
|
|
13
19
|
};
|
|
14
20
|
|
|
15
|
-
export const writeToArchive = ({ key, archive }) => {
|
|
21
|
+
export const writeToArchive = ({ key, archiveKeys, archive }) => {
|
|
16
22
|
const archivePath = path.resolve(process.cwd(), archive);
|
|
17
23
|
const archiveResult = getArchive(archive);
|
|
24
|
+
const keys = Array.from(
|
|
25
|
+
new Set([key, ...(archiveKeys || [])].filter(Boolean))
|
|
26
|
+
);
|
|
18
27
|
|
|
19
|
-
|
|
20
|
-
archiveResult.
|
|
21
|
-
|
|
28
|
+
keys.forEach((archiveKey) => {
|
|
29
|
+
if (!archiveResult.includes(archiveKey)) {
|
|
30
|
+
archiveResult.push(archiveKey);
|
|
31
|
+
}
|
|
32
|
+
});
|
|
22
33
|
|
|
23
34
|
fs.writeFileSync(archivePath, JSON.stringify(archiveResult, null, 4));
|
|
24
35
|
};
|
|
25
36
|
|
|
26
|
-
export const getIsInArchive = ({ key, archive }) => {
|
|
37
|
+
export const getIsInArchive = ({ key, archiveKeys, archive }) => {
|
|
27
38
|
const archiveResult = getArchive(archive);
|
|
28
|
-
|
|
39
|
+
const keys = [key, ...(archiveKeys || [])].filter(Boolean);
|
|
40
|
+
return keys.some((archiveKey) => archiveResult.includes(archiveKey));
|
|
29
41
|
};
|
|
30
42
|
|
|
31
43
|
export const getArchiveFilename = ({ pubDate, name, ext }) => {
|
package/bin/async.js
CHANGED
|
@@ -7,7 +7,7 @@ import { throttle } from "throttle-debounce";
|
|
|
7
7
|
import { promisify } from "util";
|
|
8
8
|
import {
|
|
9
9
|
getArchiveFilename,
|
|
10
|
-
|
|
10
|
+
getArchiveKeys,
|
|
11
11
|
getIsInArchive,
|
|
12
12
|
writeToArchive,
|
|
13
13
|
} from "./archive.js";
|
|
@@ -39,7 +39,7 @@ export const download = async (options) => {
|
|
|
39
39
|
marker,
|
|
40
40
|
url,
|
|
41
41
|
outputPath,
|
|
42
|
-
|
|
42
|
+
archiveKeys = [],
|
|
43
43
|
archive,
|
|
44
44
|
override,
|
|
45
45
|
alwaysPostprocess,
|
|
@@ -61,7 +61,11 @@ export const download = async (options) => {
|
|
|
61
61
|
return outputPath;
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
if (
|
|
64
|
+
if (
|
|
65
|
+
archive &&
|
|
66
|
+
archiveKeys.length &&
|
|
67
|
+
getIsInArchive({ archiveKeys, archive })
|
|
68
|
+
) {
|
|
65
69
|
logMessage("Download exists in archive. Skipping...");
|
|
66
70
|
return null;
|
|
67
71
|
}
|
|
@@ -149,11 +153,10 @@ export const download = async (options) => {
|
|
|
149
153
|
return null;
|
|
150
154
|
}
|
|
151
155
|
|
|
152
|
-
const
|
|
153
|
-
?
|
|
156
|
+
const finalOutputPath = trustExt
|
|
157
|
+
? outputPath
|
|
154
158
|
: correctExtensionFromMime({
|
|
155
159
|
outputPath,
|
|
156
|
-
key,
|
|
157
160
|
contentType: headResponse?.headers?.["content-type"],
|
|
158
161
|
onCorrect: (from, to) =>
|
|
159
162
|
logMessage(
|
|
@@ -170,9 +173,9 @@ export const download = async (options) => {
|
|
|
170
173
|
await onAfterDownload(finalOutputPath);
|
|
171
174
|
}
|
|
172
175
|
|
|
173
|
-
if (
|
|
176
|
+
if (archive && archiveKeys.length) {
|
|
174
177
|
try {
|
|
175
|
-
writeToArchive({
|
|
178
|
+
writeToArchive({ archiveKeys, archive });
|
|
176
179
|
} catch (error) {
|
|
177
180
|
throw new Error(`Error writing to archive: ${error.toString()}`);
|
|
178
181
|
}
|
|
@@ -245,14 +248,7 @@ export const downloadItemsAsync = async ({
|
|
|
245
248
|
marker,
|
|
246
249
|
trustExt,
|
|
247
250
|
userAgent,
|
|
248
|
-
|
|
249
|
-
prefix: archivePrefix,
|
|
250
|
-
name: getArchiveFilename({
|
|
251
|
-
name: item.title,
|
|
252
|
-
pubDate: item.pubDate,
|
|
253
|
-
ext: audioFileExt,
|
|
254
|
-
}),
|
|
255
|
-
}),
|
|
251
|
+
archiveKeys: item._archiveKeys || [],
|
|
256
252
|
maxAttempts: attempts,
|
|
257
253
|
outputPath: outputPodcastPath,
|
|
258
254
|
url: episodeAudioUrl,
|
|
@@ -264,7 +260,7 @@ export const downloadItemsAsync = async ({
|
|
|
264
260
|
override,
|
|
265
261
|
trustExt,
|
|
266
262
|
userAgent,
|
|
267
|
-
|
|
263
|
+
archiveKeys: item._episodeImage.archiveKeys || [],
|
|
268
264
|
marker: item._episodeImage.url,
|
|
269
265
|
maxAttempts: attempts,
|
|
270
266
|
outputPath: item._episodeImage.outputPath,
|
|
@@ -289,7 +285,7 @@ export const downloadItemsAsync = async ({
|
|
|
289
285
|
const finalTranscriptPath = await download({
|
|
290
286
|
archive,
|
|
291
287
|
override,
|
|
292
|
-
|
|
288
|
+
archiveKeys: item._episodeTranscript.archiveKeys || [],
|
|
293
289
|
marker: item._episodeTranscript.url,
|
|
294
290
|
maxAttempts: attempts,
|
|
295
291
|
outputPath: item._episodeTranscript.outputPath,
|
|
@@ -371,13 +367,14 @@ export const downloadItemsAsync = async ({
|
|
|
371
367
|
archive,
|
|
372
368
|
override,
|
|
373
369
|
item,
|
|
374
|
-
|
|
370
|
+
archiveKeys: getArchiveKeys({
|
|
375
371
|
prefix: archivePrefix,
|
|
376
372
|
name: getArchiveFilename({
|
|
377
373
|
pubDate: item.pubDate,
|
|
378
374
|
name: item.title,
|
|
379
375
|
ext: episodeMetaExt,
|
|
380
376
|
}),
|
|
377
|
+
guid: item.guid ? `${item.guid}-meta` : null,
|
|
381
378
|
}),
|
|
382
379
|
outputPath: outputEpisodeMetaPath,
|
|
383
380
|
});
|
package/bin/bin.js
CHANGED
|
@@ -164,10 +164,12 @@ const main = async () => {
|
|
|
164
164
|
trustExt,
|
|
165
165
|
userAgent,
|
|
166
166
|
marker: podcastImageUrl,
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
167
|
+
archiveKeys: [
|
|
168
|
+
getArchiveKey({
|
|
169
|
+
prefix: archivePrefix,
|
|
170
|
+
name: `${feed.title || "image"}${podcastImageFileExt}`,
|
|
171
|
+
}),
|
|
172
|
+
],
|
|
171
173
|
outputPath: outputImagePath,
|
|
172
174
|
url: podcastImageUrl,
|
|
173
175
|
maxAttempts: attempts,
|
package/bin/items.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import dayjs from "dayjs";
|
|
2
2
|
import path from "path";
|
|
3
|
-
import { getArchive, getArchiveFilename,
|
|
3
|
+
import { getArchive, getArchiveFilename, getArchiveKeys } from "./archive.js";
|
|
4
4
|
import { logErrorAndExit } from "./logger.js";
|
|
5
5
|
import { getItemFilename } from "./naming.js";
|
|
6
6
|
import {
|
|
@@ -48,7 +48,7 @@ export const getItemsToDownload = ({
|
|
|
48
48
|
const savedArchive = archive ? getArchive(archive) : [];
|
|
49
49
|
|
|
50
50
|
while (shouldGo(i)) {
|
|
51
|
-
const { title, pubDate, itunes } = feed.items[i];
|
|
51
|
+
const { title, pubDate, itunes, guid } = feed.items[i];
|
|
52
52
|
const actualSeasonNum = itunes?.season ? parseInt(itunes.season) : null;
|
|
53
53
|
const pubDateDay = dayjs(new Date(pubDate));
|
|
54
54
|
let isValid = true;
|
|
@@ -94,16 +94,17 @@ export const getItemsToDownload = ({
|
|
|
94
94
|
const { url: episodeAudioUrl, ext: audioFileExt } =
|
|
95
95
|
getEpisodeAudioUrlAndExt(feed.items[i], episodeSourceOrder);
|
|
96
96
|
|
|
97
|
-
const
|
|
97
|
+
const archiveKeys = getArchiveKeys({
|
|
98
98
|
prefix: archivePrefix,
|
|
99
99
|
name: getArchiveFilename({
|
|
100
100
|
pubDate,
|
|
101
101
|
name: title,
|
|
102
102
|
ext: audioFileExt,
|
|
103
103
|
}),
|
|
104
|
+
guid,
|
|
104
105
|
});
|
|
105
106
|
|
|
106
|
-
if (
|
|
107
|
+
if (archiveKeys.some((archiveKey) => savedArchive.includes(archiveKey))) {
|
|
107
108
|
isValid = false;
|
|
108
109
|
}
|
|
109
110
|
|
|
@@ -111,19 +112,21 @@ export const getItemsToDownload = ({
|
|
|
111
112
|
const item = feed.items[i];
|
|
112
113
|
item._originalIndex = i;
|
|
113
114
|
item.seasonNum = actualSeasonNum;
|
|
115
|
+
item._archiveKeys = archiveKeys;
|
|
114
116
|
|
|
115
117
|
if (includeEpisodeImages || embedMetadataFlag) {
|
|
116
118
|
const episodeImageUrl = getImageUrl(item);
|
|
117
119
|
|
|
118
120
|
if (episodeImageUrl) {
|
|
119
121
|
const episodeImageFileExt = getUrlExt(episodeImageUrl);
|
|
120
|
-
const
|
|
122
|
+
const episodeImageArchiveKeys = getArchiveKeys({
|
|
121
123
|
prefix: archivePrefix,
|
|
122
124
|
name: getArchiveFilename({
|
|
123
125
|
pubDate,
|
|
124
126
|
name: title,
|
|
125
127
|
ext: episodeImageFileExt,
|
|
126
128
|
}),
|
|
129
|
+
guid: guid ? `${guid}-image` : null,
|
|
127
130
|
});
|
|
128
131
|
|
|
129
132
|
const episodeImageName = getItemFilename({
|
|
@@ -141,7 +144,7 @@ export const getItemsToDownload = ({
|
|
|
141
144
|
item._episodeImage = {
|
|
142
145
|
url: episodeImageUrl,
|
|
143
146
|
outputPath: outputImagePath,
|
|
144
|
-
|
|
147
|
+
archiveKeys: episodeImageArchiveKeys,
|
|
145
148
|
};
|
|
146
149
|
}
|
|
147
150
|
}
|
|
@@ -154,13 +157,14 @@ export const getItemsToDownload = ({
|
|
|
154
157
|
|
|
155
158
|
if (episodeTranscriptUrl) {
|
|
156
159
|
const episodeTranscriptFileExt = getUrlExt(episodeTranscriptUrl);
|
|
157
|
-
const
|
|
160
|
+
const episodeTranscriptArchiveKeys = getArchiveKeys({
|
|
158
161
|
prefix: archivePrefix,
|
|
159
162
|
name: getArchiveFilename({
|
|
160
163
|
pubDate,
|
|
161
164
|
name: title,
|
|
162
165
|
ext: episodeTranscriptFileExt,
|
|
163
166
|
}),
|
|
167
|
+
guid: guid ? `${guid}-transcript` : null,
|
|
164
168
|
});
|
|
165
169
|
|
|
166
170
|
const episodeTranscriptName = getItemFilename({
|
|
@@ -181,7 +185,7 @@ export const getItemsToDownload = ({
|
|
|
181
185
|
item._episodeTranscript = {
|
|
182
186
|
url: episodeTranscriptUrl,
|
|
183
187
|
outputPath: outputTranscriptPath,
|
|
184
|
-
|
|
188
|
+
archiveKeys: episodeTranscriptArchiveKeys,
|
|
185
189
|
};
|
|
186
190
|
}
|
|
187
191
|
}
|
package/bin/meta.js
CHANGED
|
@@ -35,11 +35,15 @@ export const writeItemMeta = ({
|
|
|
35
35
|
marker,
|
|
36
36
|
outputPath,
|
|
37
37
|
item,
|
|
38
|
-
|
|
38
|
+
archiveKeys,
|
|
39
39
|
archive,
|
|
40
40
|
override,
|
|
41
41
|
}) => {
|
|
42
|
-
if (
|
|
42
|
+
if (
|
|
43
|
+
archive &&
|
|
44
|
+
archiveKeys?.length &&
|
|
45
|
+
getIsInArchive({ archiveKeys, archive })
|
|
46
|
+
) {
|
|
43
47
|
logMessage(`${marker} | Episode metadata exists in archive. Skipping...`);
|
|
44
48
|
return;
|
|
45
49
|
}
|
|
@@ -53,9 +57,13 @@ export const writeItemMeta = ({
|
|
|
53
57
|
logMessage(`${marker} | Episode metadata exists locally. Skipping...`);
|
|
54
58
|
}
|
|
55
59
|
|
|
56
|
-
if (
|
|
60
|
+
if (
|
|
61
|
+
archive &&
|
|
62
|
+
archiveKeys?.length &&
|
|
63
|
+
!getIsInArchive({ archiveKeys, archive })
|
|
64
|
+
) {
|
|
57
65
|
try {
|
|
58
|
-
writeToArchive({
|
|
66
|
+
writeToArchive({ archiveKeys, archive });
|
|
59
67
|
} catch (error) {
|
|
60
68
|
throw new Error("Error writing to archive", error);
|
|
61
69
|
}
|
package/bin/util.js
CHANGED
|
@@ -242,7 +242,6 @@ const getMimeCategory = (mime) => {
|
|
|
242
242
|
|
|
243
243
|
export const correctExtensionFromMime = ({
|
|
244
244
|
outputPath,
|
|
245
|
-
key,
|
|
246
245
|
contentType,
|
|
247
246
|
onCorrect,
|
|
248
247
|
}) => {
|
|
@@ -250,32 +249,28 @@ export const correctExtensionFromMime = ({
|
|
|
250
249
|
const mimeExt = mimeType ? getExtFromMime(mimeType) : null;
|
|
251
250
|
|
|
252
251
|
if (!mimeExt) {
|
|
253
|
-
return
|
|
252
|
+
return outputPath;
|
|
254
253
|
}
|
|
255
254
|
|
|
256
255
|
const currentExt = path.extname(outputPath);
|
|
257
256
|
if (mimeExt === currentExt) {
|
|
258
|
-
return
|
|
257
|
+
return outputPath;
|
|
259
258
|
}
|
|
260
259
|
|
|
261
260
|
const currentCategory = getExtCategory(currentExt);
|
|
262
261
|
const mimeCategory = getMimeCategory(mimeType);
|
|
263
262
|
|
|
264
263
|
if (currentCategory && mimeCategory && currentCategory !== mimeCategory) {
|
|
265
|
-
return
|
|
264
|
+
return outputPath;
|
|
266
265
|
}
|
|
267
266
|
|
|
268
267
|
const basePath = currentExt
|
|
269
268
|
? outputPath.slice(0, -currentExt.length)
|
|
270
269
|
: outputPath;
|
|
271
|
-
const baseKey = key && currentExt ? key.slice(0, -currentExt.length) : key;
|
|
272
270
|
|
|
273
271
|
onCorrect?.(currentExt || "(none)", mimeExt);
|
|
274
272
|
|
|
275
|
-
return
|
|
276
|
-
outputPath: basePath + mimeExt,
|
|
277
|
-
key: baseKey ? baseKey + mimeExt : null,
|
|
278
|
-
};
|
|
273
|
+
return basePath + mimeExt;
|
|
279
274
|
};
|
|
280
275
|
|
|
281
276
|
export const VALID_AUDIO_EXTS = [
|