podcast-dl 11.4.0 → 11.5.0

package/LICENSE

@@ -1,21 +1,21 @@
-MIT License
-
-Copyright (c) 2020 Joshua Pohl
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+MIT License
+
+Copyright (c) 2020 Joshua Pohl
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -61,6 +61,7 @@ Type values surrounded in square brackets (`[]`) can be used as boolean options
 | --parser-config                   | String              | false    | Path to JSON file that will be parsed and used to override the default config passed to [rss-parser](https://github.com/rbren/rss-parser#xml-options).                                                                        |
 | --user-agent                      | String              | false    | Specify custom user agent string for HTTP requests. Defaults to a Chrome user agent if not specified.                                                                                                                         |
 | --proxy                           |                     | false    | Enable proxy support. Specify environment variables listed by [global-agent](https://github.com/gajus/global-agent#environment-variables).                                                                                    |
+| --trust-ext                       |                     | false    | Trust file extension from URL, skip MIME-based correction.                                                                                                                                                                    |
 | --help                            |                     | false    | Output usage information.                                                                                                                                                                                                     |
 
 ## Archive

package/bin/archive.js

@@ -1,39 +1,39 @@
-import dayjs from "dayjs";
-import fs from "fs";
-import path from "path";
-import { getJsonFile } from "./util.js";
-
-export const getArchiveKey = ({ prefix, name }) => {
-  return `${prefix}-${name}`;
-};
-
-export const getArchive = (archive) => {
-  const archiveContent = getJsonFile(archive);
-  return archiveContent === null ? [] : archiveContent;
-};
-
-export const writeToArchive = ({ key, archive }) => {
-  const archivePath = path.resolve(process.cwd(), archive);
-  const archiveResult = getArchive(archive);
-
-  if (!archiveResult.includes(key)) {
-    archiveResult.push(key);
-  }
-
-  fs.writeFileSync(archivePath, JSON.stringify(archiveResult, null, 4));
-};
-
-export const getIsInArchive = ({ key, archive }) => {
-  const archiveResult = getArchive(archive);
-  return archiveResult.includes(key);
-};
-
-export const getArchiveFilename = ({ pubDate, name, ext }) => {
-  const formattedPubDate = pubDate
-    ? dayjs(new Date(pubDate)).format("YYYYMMDD")
-    : null;
-
-  const baseName = formattedPubDate ? `${formattedPubDate}-${name}` : name;
-
-  return `${baseName}${ext}`;
-};
+import dayjs from "dayjs";
+import fs from "fs";
+import path from "path";
+import { getJsonFile } from "./util.js";
+
+export const getArchiveKey = ({ prefix, name }) => {
+  return `${prefix}-${name}`;
+};
+
+export const getArchive = (archive) => {
+  const archiveContent = getJsonFile(archive);
+  return archiveContent === null ? [] : archiveContent;
+};
+
+export const writeToArchive = ({ key, archive }) => {
+  const archivePath = path.resolve(process.cwd(), archive);
+  const archiveResult = getArchive(archive);
+
+  if (!archiveResult.includes(key)) {
+    archiveResult.push(key);
+  }
+
+  fs.writeFileSync(archivePath, JSON.stringify(archiveResult, null, 4));
+};
+
+export const getIsInArchive = ({ key, archive }) => {
+  const archiveResult = getArchive(archive);
+  return archiveResult.includes(key);
+};
+
+export const getArchiveFilename = ({ pubDate, name, ext }) => {
+  const formattedPubDate = pubDate
+    ? dayjs(new Date(pubDate)).format("YYYYMMDD")
+    : null;
+
+  const baseName = formattedPubDate ? `${formattedPubDate}-${name}` : name;
+
+  return `${baseName}${ext}`;
+};

package/bin/async.js

@@ -22,6 +22,7 @@ import {
 import { writeItemMeta } from "./meta.js";
 import { getItemFilename } from "./naming.js";
 import {
+  correctExtensionFromMime,
   getEpisodeAudioUrlAndExt,
   getTempPath,
   prepareOutputPath,
@@ -45,6 +46,7 @@ export const download = async (options) => {
     onAfterDownload,
     attempt = 1,
     maxAttempts = 3,
+    trustExt,
     userAgent = USER_AGENT,
   } = options;
 
@@ -56,12 +58,12 @@ export const download = async (options) => {
       await onAfterDownload();
     }
 
-    return;
+    return outputPath;
   }
 
   if (key && archive && getIsInArchive({ key, archive })) {
     logMessage("Download exists in archive. Skipping...");
-    return;
+    return null;
   }
 
   let headResponse = null;
@@ -75,7 +77,7 @@ export const download = async (options) => {
         "user-agent": userAgent,
       },
     });
-  } catch (error) {
+  } catch {
     // unable to retrieve head response
   }
 
@@ -125,7 +127,7 @@ export const download = async (options) => {
     if (attempt <= maxAttempts) {
       logMessage(`Download attempt #${attempt} failed. Retrying...`);
 
-      await download({
+      return await download({
         ...options,
         attempt: attempt + 1,
       });
@@ -144,24 +146,39 @@ export const download = async (options) => {
       LOG_LEVELS.important
     );
 
-    return;
+    return null;
   }
 
-  fs.renameSync(tempOutputPath, outputPath);
+  const { outputPath: finalOutputPath, key: finalKey } = trustExt
+    ? { outputPath, key }
+    : correctExtensionFromMime({
+        outputPath,
+        key,
+        contentType: headResponse?.headers?.["content-type"],
+        onCorrect: (from, to) =>
+          logMessage(
+            `Correcting extension: ${from} --> ${to}`,
+            LOG_LEVELS.important
+          ),
+      });
+
+  fs.renameSync(tempOutputPath, finalOutputPath);
 
   logMessage("Download complete!");
 
   if (onAfterDownload) {
-    await onAfterDownload();
+    await onAfterDownload(finalOutputPath);
   }
 
-  if (key && archive) {
+  if (finalKey && archive) {
     try {
-      writeToArchive({ key, archive });
+      writeToArchive({ key: finalKey, archive });
     } catch (error) {
       throw new Error(`Error writing to archive: ${error.toString()}`);
     }
   }
+
+  return finalOutputPath;
 };
 
 export const downloadItemsAsync = async ({
@@ -185,6 +202,7 @@ export const downloadItemsAsync = async ({
   alwaysPostprocess,
   targetItems,
   threads = 1,
+  trustExt,
   userAgent = USER_AGENT,
 }) => {
   let numEpisodesDownloaded = 0;
@@ -224,6 +242,7 @@ export const downloadItemsAsync = async ({
         override,
         alwaysPostprocess,
         marker,
+        trustExt,
         userAgent,
         key: getArchiveKey({
           prefix: archivePrefix,
@@ -236,12 +255,13 @@ export const downloadItemsAsync = async ({
         maxAttempts: attempts,
         outputPath: outputPodcastPath,
         url: episodeAudioUrl,
-        onAfterDownload: async () => {
+        onAfterDownload: async (finalEpisodePath) => {
           if (item._episodeImage) {
             try {
-              await download({
+              const finalImagePath = await download({
                 archive,
                 override,
+                trustExt,
                 userAgent,
                 key: item._episodeImage.key,
                 marker: item._episodeImage.url,
@@ -249,6 +269,10 @@ export const downloadItemsAsync = async ({
                 outputPath: item._episodeImage.outputPath,
                 url: item._episodeImage.url,
               });
+
+              if (finalImagePath) {
+                item._episodeImage.outputPath = finalImagePath;
+              }
             } catch (error) {
               hasErrors = true;
               logError(
@@ -261,16 +285,21 @@ export const downloadItemsAsync = async ({
 
           if (item._episodeTranscript) {
             try {
-              await download({
+              const finalTranscriptPath = await download({
                 archive,
                 override,
                 key: item._episodeTranscript.key,
                 marker: item._episodeTranscript.url,
                 maxAttempts: attempts,
                 outputPath: item._episodeTranscript.outputPath,
+                trustExt,
                 url: item._episodeTranscript.url,
                 userAgent,
               });
+
+              if (finalTranscriptPath) {
+                item._episodeTranscript.outputPath = finalTranscriptPath;
+              }
             } catch (error) {
               hasErrors = true;
               logError(
@@ -292,7 +321,7 @@ export const downloadItemsAsync = async ({
               bitrate,
               mono,
               itemIndex: item._originalIndex,
-              outputPath: outputPodcastPath,
+              outputPath: finalEpisodePath,
               episodeImageOutputPath: hasEpisodeImage
                 ? item._episodeImage.outputPath
                 : undefined,
@@ -310,7 +339,7 @@ export const downloadItemsAsync = async ({
             await runExec({
               exec,
               basePath,
-              outputPodcastPath,
+              outputPodcastPath: finalEpisodePath,
               episodeFilename,
               episodeAudioUrl,
             });

package/bin/bin.js

@@ -63,6 +63,7 @@ const {
   addMp3Metadata: addMp3MetadataFlag,
   adjustBitrate: bitrate,
   season,
+  trustExt,
 } = opts;
 
 let { archive } = opts;
@@ -148,10 +149,7 @@ const main = async () => {
       const podcastImageFileExt = getUrlExt(podcastImageUrl);
       const outputImagePath = _path.resolve(
         basePath,
-        getSimpleFilename(
-          feed.title ? feed.title : "image",
-          feed.title ? `.image${podcastImageFileExt}` : podcastImageFileExt
-        )
+        getSimpleFilename(feed.title || "image", podcastImageFileExt)
       );
 
       try {
@@ -159,13 +157,12 @@ const main = async () => {
         await download({
           archive,
           override,
+          trustExt,
           userAgent,
           marker: podcastImageUrl,
           key: getArchiveKey({
             prefix: archivePrefix,
-            name: `${
-              feed.title ? `${feed.title}.image` : "image"
-            }${podcastImageFileExt}`,
+            name: `${feed.title || "image"}${podcastImageFileExt}`,
           }),
           outputPath: outputImagePath,
           url: podcastImageUrl,
@@ -262,6 +259,7 @@ const main = async () => {
     alwaysPostprocess,
     targetItems,
     threads,
+    trustExt,
     userAgent,
   });
 

package/bin/commander.js

@@ -191,7 +191,8 @@ export const setupCommander = (program) => {
       "path to JSON config to override RSS parser"
     )
     .option("--proxy", "enable proxy support via global-agent")
-    .option("--user-agent <string>", "specify custom user agent string");
+    .option("--user-agent <string>", "specify custom user agent string")
+    .option("--trust-ext", "trust file extension, skip MIME-based correction");
 
   program.parse();
 

package/bin/exec.js

@@ -1,30 +1,30 @@
-import { exec } from "child_process";
-import util from "util";
-import { escapeArgForShell } from "./util.js";
-
-export const execWithPromise = util.promisify(exec);
-
-export const runExec = async ({
-  exec,
-  basePath,
-  outputPodcastPath,
-  episodeFilename,
-  episodeAudioUrl,
-}) => {
-  const episodeFilenameBase = episodeFilename.substring(
-    0,
-    episodeFilename.lastIndexOf(".")
-  );
-
-  const execCmd = exec
-    .replace(/{{episode_path}}/g, escapeArgForShell(outputPodcastPath))
-    .replace(/{{episode_path_base}}/g, escapeArgForShell(basePath))
-    .replace(/{{episode_filename}}/g, escapeArgForShell(episodeFilename))
-    .replace(
-      /{{episode_filename_base}}/g,
-      escapeArgForShell(episodeFilenameBase)
-    )
-    .replace(/{{url}}/g, escapeArgForShell(episodeAudioUrl));
-
-  await execWithPromise(execCmd, { stdio: "ignore" });
-};
+import { exec } from "child_process";
+import util from "util";
+import { escapeArgForShell } from "./util.js";
+
+export const execWithPromise = util.promisify(exec);
+
+export const runExec = async ({
+  exec,
+  basePath,
+  outputPodcastPath,
+  episodeFilename,
+  episodeAudioUrl,
+}) => {
+  const episodeFilenameBase = episodeFilename.substring(
+    0,
+    episodeFilename.lastIndexOf(".")
+  );
+
+  const execCmd = exec
+    .replace(/{{episode_path}}/g, escapeArgForShell(outputPodcastPath))
+    .replace(/{{episode_path_base}}/g, escapeArgForShell(basePath))
+    .replace(/{{episode_filename}}/g, escapeArgForShell(episodeFilename))
+    .replace(
+      /{{episode_filename_base}}/g,
+      escapeArgForShell(episodeFilenameBase)
+    )
+    .replace(/{{url}}/g, escapeArgForShell(episodeAudioUrl));
+
+  await execWithPromise(execCmd, { stdio: "ignore" });
+};

package/bin/ffmpeg.js

@@ -1,105 +1,105 @@
-import dayjs from "dayjs";
-import fs from "fs";
-import { execWithPromise } from "./exec.js";
-import { LOG_LEVELS, logMessage } from "./logger.js";
-import { escapeArgForShell, isWin } from "./util.js";
-
-export const runFfmpeg = async ({
-  feed,
-  item,
-  itemIndex,
-  outputPath,
-  episodeImageOutputPath,
-  bitrate,
-  mono,
-  addMp3Metadata,
-  ext,
-}) => {
-  if (!fs.existsSync(outputPath)) {
-    return;
-  }
-
-  const shouldEmbedImage = addMp3Metadata && episodeImageOutputPath;
-  let command = `ffmpeg -loglevel quiet -i ${escapeArgForShell(outputPath)}`;
-
-  if (shouldEmbedImage) {
-    command += ` -i ${escapeArgForShell(episodeImageOutputPath)}`;
-  }
-
-  if (bitrate) {
-    command += ` -b:a ${bitrate}`;
-  }
-
-  if (mono) {
-    command += " -ac 1";
-  }
-
-  if (addMp3Metadata) {
-    const album = feed.title || "";
-    const artist = item.itunes?.author || item.author || "";
-    const title = item.title || "";
-    const subtitle = item.itunes?.subtitle || "";
-    const comment = item.contentSnippet || item.content || "";
-    const disc = item.itunes?.season || "";
-    const track = item.itunes?.episode || `${feed.items.length - itemIndex}`;
-    const episodeType = item.itunes?.episodeType || "";
-    const date = item.pubDate
-      ? dayjs(new Date(item.pubDate)).format("YYYY-MM-DD")
-      : "";
-
-    const metaKeysToValues = {
-      album,
-      artist,
-      album_artist: artist,
-      title,
-      disc,
-      track,
-      "episode-type": episodeType,
-      date,
-    };
-
-    if (!isWin) {
-      // Due to limited escape options, these metadata fields often break in Windows
-      metaKeysToValues.comment = comment;
-      metaKeysToValues.subtitle = subtitle
-    }
-
-    const metadataString = Object.keys(metaKeysToValues)
-      .map((key) => {
-        if (!metaKeysToValues[key]) {
-          return null;
-        }
-
-        const argValue = escapeArgForShell(metaKeysToValues[key]);
-
-        return argValue ? `-metadata ${key}=${argValue}` : null;
-      })
-      .filter((segment) => !!segment)
-      .join(" ");
-
-    command += ` -map_metadata 0 ${metadataString} -codec copy`;
-  }
-
-  if (shouldEmbedImage) {
-    command += ` -map 0 -map 1`;
-  } else {
-    command += ` -map 0`;
-  }
-
-  const tmpMp3Path = `${outputPath}.tmp${ext}`;
-  command += ` ${escapeArgForShell(tmpMp3Path)}`;
-  logMessage("Running command: " + command, LOG_LEVELS.debug);
-
-  try {
-    await execWithPromise(command, { stdio: "ignore" });
-  } catch (error) {
-    if (fs.existsSync(tmpMp3Path)) {
-      fs.unlinkSync(tmpMp3Path);
-    }
-
-    throw error;
-  }
-
-  fs.unlinkSync(outputPath);
-  fs.renameSync(tmpMp3Path, outputPath);
-};
+import dayjs from "dayjs";
+import fs from "fs";
+import { execWithPromise } from "./exec.js";
+import { LOG_LEVELS, logMessage } from "./logger.js";
+import { escapeArgForShell, isWin } from "./util.js";
+
+export const runFfmpeg = async ({
+  feed,
+  item,
+  itemIndex,
+  outputPath,
+  episodeImageOutputPath,
+  bitrate,
+  mono,
+  addMp3Metadata,
+  ext,
+}) => {
+  if (!fs.existsSync(outputPath)) {
+    return;
+  }
+
+  const shouldEmbedImage = addMp3Metadata && episodeImageOutputPath;
+  let command = `ffmpeg -loglevel quiet -i ${escapeArgForShell(outputPath)}`;
+
+  if (shouldEmbedImage) {
+    command += ` -i ${escapeArgForShell(episodeImageOutputPath)}`;
+  }
+
+  if (bitrate) {
+    command += ` -b:a ${bitrate}`;
+  }
+
+  if (mono) {
+    command += " -ac 1";
+  }
+
+  if (addMp3Metadata) {
+    const album = feed.title || "";
+    const artist = item.itunes?.author || item.author || "";
+    const title = item.title || "";
+    const subtitle = item.itunes?.subtitle || "";
+    const comment = item.contentSnippet || item.content || "";
+    const disc = item.itunes?.season || "";
+    const track = item.itunes?.episode || `${feed.items.length - itemIndex}`;
+    const episodeType = item.itunes?.episodeType || "";
+    const date = item.pubDate
+      ? dayjs(new Date(item.pubDate)).format("YYYY-MM-DD")
+      : "";
+
+    const metaKeysToValues = {
+      album,
+      artist,
+      album_artist: artist,
+      title,
+      disc,
+      track,
+      "episode-type": episodeType,
+      date,
+    };
+
+    if (!isWin) {
+      // Due to limited escape options, these metadata fields often break in Windows
+      metaKeysToValues.comment = comment;
+      metaKeysToValues.subtitle = subtitle
+    }
+
+    const metadataString = Object.keys(metaKeysToValues)
+      .map((key) => {
+        if (!metaKeysToValues[key]) {
+          return null;
+        }
+
+        const argValue = escapeArgForShell(metaKeysToValues[key]);
+
+        return argValue ? `-metadata ${key}=${argValue}` : null;
+      })
+      .filter((segment) => !!segment)
+      .join(" ");
+
+    command += ` -map_metadata 0 ${metadataString} -codec copy`;
+  }
+
+  if (shouldEmbedImage) {
+    command += ` -map 0 -map 1`;
+  } else {
+    command += ` -map 0`;
+  }
+
+  const tmpMp3Path = `${outputPath}.tmp${ext}`;
+  command += ` ${escapeArgForShell(tmpMp3Path)}`;
+  logMessage("Running command: " + command, LOG_LEVELS.debug);
+
+  try {
+    await execWithPromise(command, { stdio: "ignore" });
+  } catch (error) {
+    if (fs.existsSync(tmpMp3Path)) {
+      fs.unlinkSync(tmpMp3Path);
+    }
+
+    throw error;
+  }
+
+  fs.unlinkSync(outputPath);
+  fs.renameSync(tmpMp3Path, outputPath);
+};