podcast-dl 6.1.0 → 7.1.0

package/bin/bin.js

@@ -1,42 +1,36 @@
 #!/usr/bin/env node
 
-const fs = require("fs");
-const _path = require("path");
-const _url = require("url");
-const commander = require("commander");
-
-const { version } = require("../package.json");
-const {
-  download,
+import fs from "fs";
+import _path from "path";
+import commander from "commander";
+import { createRequire } from "module";
+import pluralize from "pluralize";
+
+import { download } from "./async.js";
+import {
   getArchiveKey,
-  getEpisodeAudioUrlAndExt,
   getFeed,
   getImageUrl,
   getItemsToDownload,
   getUrlExt,
   logFeedInfo,
-  logItemInfo,
   logItemsList,
   writeFeedMeta,
-  writeItemMeta,
-  runExec,
   ITEM_LIST_FORMATS,
-  runFfmpeg,
-} = require("./util");
-const { createParseNumber } = require("./validate");
-const {
+} from "./util.js";
+import { createParseNumber, hasFfmpeg } from "./validate.js";
+import {
   ERROR_STATUSES,
   LOG_LEVELS,
   logMessage,
   logError,
   logErrorAndExit,
-} = require("./logger");
-const {
-  getFilename,
-  getFolderName,
-  getArchiveFilename,
-  getSafeName,
-} = require("./naming");
+} from "./logger.js";
+import { getFolderName, getSafeName } from "./naming.js";
+import { downloadItemsAsync } from "./async.js";
+
+const require = createRequire(import.meta.url);
+const { version } = require("../package.json");
 
 commander
   .version(version)
@@ -56,10 +50,7 @@ commander
     "--include-episode-meta",
     "write out individual episode metadata to json"
   )
-  .option(
-    "--ignore-episode-images",
-    "ignore downloading found images from --include-episode-meta"
-  )
+  .option("--include-episode-images", "download found episode images")
   .option(
     "--offset <number>",
     "offset episode to start downloading from (most recent = 0)",
@@ -85,13 +76,19 @@ commander
   )
   .option(
     "--add-mp3-metadata",
-    "attempts to add a base level of metadata to .mp3 files using ffmpeg"
+    "attempts to add a base level of metadata to .mp3 files using ffmpeg",
+    hasFfmpeg
   )
   .option(
     "--adjust-bitrate <string>",
-    "attempts to adjust bitrate of .mp3 files using ffmpeg"
+    "attempts to adjust bitrate of .mp3 files using ffmpeg",
+    hasFfmpeg
+  )
+  .option(
+    "--mono",
+    "attempts to force .mp3 files into mono using ffmpeg",
+    hasFfmpeg
   )
-  .option("--mono", "attempts to force .mp3 files into mono using ffmpeg")
   .option("--override", "override local files on collision")
   .option("--reverse", "download episodes in reverse order")
   .option("--info", "print retrieved podcast info instead of downloading")
@@ -115,6 +112,16 @@ commander
     "--exec <string>",
     "Execute a command after each episode is downloaded"
   )
+  .option(
+    "--threads <number>",
+    "the number of downloads that can happen concurrently",
+    createParseNumber({ min: 1, max: 32, name: "threads" }),
+    1
+  )
+  .option(
+    "--filter-url-tracking",
+    "attempts to extract the direct download link of an episode if detected (experimental)"
+  )
   .parse(process.argv);
 
 const {
@@ -123,7 +130,7 @@ const {
   episodeTemplate,
   includeMeta,
   includeEpisodeMeta,
-  ignoreEpisodeImages,
+  includeEpisodeImages,
   offset,
   limit,
   episodeRegex,
@@ -135,6 +142,8 @@ const {
   list,
   exec,
   mono,
+  threads,
+  filterUrlTracking,
   addMp3Metadata: addMp3MetadataFlag,
   adjustBitrate: bitrate,
 } = commander;
@@ -146,7 +155,7 @@ const main = async () => {
     logErrorAndExit("No URL provided");
   }
 
-  const { hostname, pathname } = _url.parse(url);
+  const { hostname, pathname } = new URL(url);
   const archiveUrl = `${hostname}${pathname}`;
   const feed = await getFeed(url);
   const basePath = _path.resolve(
@@ -154,9 +163,7 @@ const main = async () => {
     getFolderName({ feed, template: outDir })
   );
 
-  if (info) {
-    logFeedInfo(feed);
-  }
+  logFeedInfo(feed);
 
   if (list) {
     if (feed.items && feed.items.length) {
@@ -207,32 +214,35 @@ const main = async () => {
       );
 
       try {
-        logMessage("Saving podcast image");
+        logMessage("\nDownloading podcast image...");
         await download({
           archive,
           override,
+          marker: podcastImageUrl,
           key: getArchiveKey({ prefix: archiveUrl, name: podcastImageName }),
           outputPath: outputImagePath,
           url: podcastImageUrl,
         });
       } catch (error) {
-        logError("Unable to download episode image", error);
+        logError("Unable to download podcast image", error);
       }
-    } else {
-      logMessage("Unable to find podcast image");
     }
 
     const outputMetaName = `${feed.title ? `${feed.title}.meta` : "meta"}.json`;
     const outputMetaPath = _path.resolve(basePath, getSafeName(outputMetaName));
 
-    logMessage("Saving podcast metadata");
-    writeFeedMeta({
-      archive,
-      override,
-      feed,
-      key: getArchiveKey({ prefix: archiveUrl, name: outputMetaName }),
-      outputPath: outputMetaPath,
-    });
+    try {
+      logMessage("\nSaving podcast metadata...");
+      writeFeedMeta({
+        archive,
+        override,
+        feed,
+        key: getArchiveKey({ prefix: archiveUrl, name: outputMetaName }),
+        outputPath: outputMetaPath,
+      });
+    } catch (error) {
+      logError("Unable to save podcast metadata", error);
+    }
   }
 
   if (!feed.items || feed.items.length === 0) {
@@ -244,6 +254,9 @@ const main = async () => {
   }
 
   const targetItems = getItemsToDownload({
+    archive,
+    archiveUrl,
+    basePath,
     feed,
     limit,
     offset,
@@ -251,159 +264,41 @@ const main = async () => {
     after,
     before,
     episodeRegex,
+    episodeTemplate,
+    includeEpisodeImages,
   });
 
   if (!targetItems.length) {
     logErrorAndExit("No episodes found with provided criteria to download");
   }
 
-  const nextItem = () => {
-    counter += 1;
-    logMessage("");
-  };
-
-  const episodeText = targetItems.length === 1 ? "episode" : "episodes";
-  logMessage(`Starting download of ${targetItems.length} ${episodeText}\n`);
-
-  let counter = 1;
-  let episodesDownloadedCounter = 0;
-
-  for (const item of targetItems) {
-    logMessage(`${counter} of ${targetItems.length}`);
-
-    const { url: episodeAudioUrl, ext: audioFileExt } =
-      getEpisodeAudioUrlAndExt(item);
-
-    if (!episodeAudioUrl) {
-      logItemInfo(item, LOG_LEVELS.critical);
-      logError("Unable to find episode download URL. Skipping");
-      nextItem();
-      continue;
-    }
-
-    const episodeFilename = getFilename({
-      item,
-      feed,
-      url: episodeAudioUrl,
-      ext: audioFileExt,
-      template: episodeTemplate,
-    });
-    const outputPodcastPath = _path.resolve(basePath, episodeFilename);
-
-    try {
-      await download({
-        archive,
-        override,
-        key: getArchiveKey({
-          prefix: archiveUrl,
-          name: getArchiveFilename({
-            name: item.title,
-            pubDate: item.pubDate,
-            ext: audioFileExt,
-          }),
-        }),
-        outputPath: outputPodcastPath,
-        url: episodeAudioUrl,
-        onSkip: () => {
-          logItemInfo(item);
-        },
-        onBeforeDownload: () => {
-          logItemInfo(item, LOG_LEVELS.important);
-        },
-        onAfterDownload: () => {
-          if (addMp3MetadataFlag || bitrate || mono) {
-            runFfmpeg({
-              feed,
-              item,
-              bitrate,
-              mono,
-              itemIndex: item._originalIndex,
-              outputPath: outputPodcastPath,
-            });
-          }
-
-          if (exec) {
-            runExec({ exec, outputPodcastPath, episodeFilename });
-          }
-
-          episodesDownloadedCounter += 1;
-        },
-      });
-    } catch (error) {
-      logError("Unable to download episode", error);
-    }
-
-    if (includeEpisodeMeta) {
-      if (!ignoreEpisodeImages) {
-        const episodeImageUrl = getImageUrl(item);
-
-        if (episodeImageUrl) {
-          const episodeImageFileExt = getUrlExt(episodeImageUrl);
-          const episodeImageName = getFilename({
-            item,
-            feed,
-            url: episodeAudioUrl,
-            ext: episodeImageFileExt,
-            template: episodeTemplate,
-          });
-          const outputImagePath = _path.resolve(basePath, episodeImageName);
-
-          logMessage("Saving episode image");
-          try {
-            await download({
-              archive,
-              override,
-              key: getArchiveKey({
-                prefix: archiveUrl,
-                name: getArchiveFilename({
-                  pubDate: item.pubDate,
-                  name: item.title,
-                  ext: episodeImageFileExt,
-                }),
-              }),
-              outputPath: outputImagePath,
-              url: episodeImageUrl,
-            });
-          } catch (error) {
-            logError("Unable to download episode image", error);
-          }
-        } else {
-          logMessage("Unable to find episode image URL");
-        }
-      }
-
-      const episodeMetaExt = ".meta.json";
-      const episodeMetaName = getFilename({
-        item,
-        feed,
-        url: episodeAudioUrl,
-        ext: episodeMetaExt,
-        template: episodeTemplate,
-      });
-      const outputEpisodeMetaPath = _path.resolve(basePath, episodeMetaName);
+  logMessage(
+    `\nStarting download of ${pluralize("episode", targetItems.length, true)}\n`
+  );
 
-      logMessage("Saving episode metadata");
-      writeItemMeta({
-        archive,
-        override,
-        item,
-        key: getArchiveKey({
-          prefix: archiveUrl,
-          name: getArchiveFilename({
-            pubDate: item.pubDate,
-            name: item.title,
-            ext: episodeMetaExt,
-          }),
-        }),
-        outputPath: outputEpisodeMetaPath,
-      });
-    }
+  const { numEpisodesDownloaded, hasErrors } = await downloadItemsAsync({
+    addMp3MetadataFlag,
+    archive,
+    archiveUrl,
+    basePath,
+    bitrate,
+    episodeTemplate,
+    exec,
+    feed,
+    includeEpisodeMeta,
+    mono,
+    override,
+    targetItems,
+    threads,
+    filterUrlTracking,
+  });
 
-    nextItem();
+  if (numEpisodesDownloaded === 0) {
+    process.exit(ERROR_STATUSES.nothingDownloaded);
   }
 
-  if (episodesDownloadedCounter === 0) {
-    process.exit(ERROR_STATUSES.nothingDownloaded);
+  if (hasErrors) {
+    process.exit(ERROR_STATUSES.completedWithErrors);
   }
 };
 

package/bin/logger.js

@@ -1,6 +1,7 @@
 const ERROR_STATUSES = {
   general: 1,
   nothingDownloaded: 2,
+  completedWithErrors: 3,
 };
 
 const LOG_LEVEL_TYPES = {
@@ -25,7 +26,7 @@ const getShouldOutputProgressIndicator = () => {
   );
 };
 
-const logMessage = (message, logLevel = 1) => {
+const logMessage = (message = "", logLevel = 1) => {
   if (
     !process.env.LOG_LEVEL ||
     process.env.LOG_LEVEL === LOG_LEVEL_TYPES.debug ||
@@ -48,6 +49,16 @@ const logMessage = (message, logLevel = 1) => {
   }
 };
 
+const getLogMessageWithMarker = (marker) => {
+  return (message, logLevel) => {
+    if (marker) {
+      logMessage(`${marker} | ${message}`, logLevel);
+    } else {
+      logMessage(message, logLevel);
+    }
+  };
+};
+
 const logError = (msg, error) => {
   if (process.env.LOG_LEVEL === LOG_LEVEL_TYPES.silent) {
     return;
@@ -70,9 +81,10 @@ const logErrorAndExit = (msg, error) => {
   process.exit(ERROR_STATUSES.general);
 };
 
-module.exports = {
+export {
   ERROR_STATUSES,
   getShouldOutputProgressIndicator,
+  getLogMessageWithMarker,
   LOG_LEVELS,
   logMessage,
   logError,

package/bin/naming.js

@@ -1,5 +1,5 @@
-const filenamify = require("filenamify");
-const dayjs = require("dayjs");
+import filenamify from "filenamify";
+import dayjs from "dayjs";
 
 const INVALID_CHAR_REPLACE = "_";
 const MAX_LENGTH_FILENAME = 255;
@@ -67,9 +67,4 @@ const getArchiveFilename = ({ pubDate, name, ext }) => {
   return getSafeName(`${baseName}${ext}`);
 };
 
-module.exports = {
-  getArchiveFilename,
-  getFilename,
-  getFolderName,
-  getSafeName,
-};
+export { getArchiveFilename, getFilename, getFolderName, getSafeName };