podcast-dl 6.1.0 → 7.1.0

package/bin/util.js

@@ -1,26 +1,98 @@
-const _url = require("url");
-const rssParser = require("rss-parser");
-const { promisify } = require("util");
-const stream = require("stream");
-const path = require("path");
-const fs = require("fs");
-const got = require("got");
-const dayjs = require("dayjs");
-const { execSync } = require("child_process");
-
-const {
-  getShouldOutputProgressIndicator,
-  logMessage,
-  logError,
-  logErrorAndExit,
-  LOG_LEVELS,
-} = require("./logger");
-
-const pipeline = promisify(stream.pipeline);
+import rssParser from "rss-parser";
+import path from "path";
+import fs from "fs";
+import dayjs from "dayjs";
+import got from "got";
+import util from "util";
+import { exec } from "child_process";
+
+import { logErrorAndExit, logMessage } from "./logger.js";
+import { getArchiveFilename, getFilename } from "./naming.js";
+
+const execWithPromise = util.promisify(exec);
+
 const parser = new rssParser({
   defaultRSS: 2.0,
 });
 
+const getArchiveKey = ({ prefix, name }) => {
+  return `${prefix}-${name}`;
+};
+
+const getArchive = (archive) => {
+  const archivePath = path.resolve(process.cwd(), archive);
+
+  if (!fs.existsSync(archivePath)) {
+    return [];
+  }
+
+  return JSON.parse(fs.readFileSync(archivePath));
+};
+
+const writeToArchive = ({ key, archive }) => {
+  const archivePath = path.resolve(process.cwd(), archive);
+  const archiveResult = getArchive(archive);
+
+  if (!archiveResult.includes(key)) {
+    archiveResult.push(key);
+  }
+
+  fs.writeFileSync(archivePath, JSON.stringify(archiveResult, null, 4));
+};
+
+const getIsInArchive = ({ key, archive }) => {
+  const archiveResult = getArchive(archive);
+  return archiveResult.includes(key);
+};
+
+const getPossibleUrlEmbeds = (url, maxAmount = 5) => {
+  const fullUrl = new URL(url);
+  const possibleStartIndexes = [];
+
+  for (let i = 0; i < fullUrl.pathname.length; i++) {
+    if (fullUrl.pathname[i] === "/") {
+      possibleStartIndexes.push(i);
+    }
+  }
+
+  const possibleEmbedChoices = possibleStartIndexes.map((startIndex) => {
+    let possibleEmbed = fullUrl.pathname.slice(startIndex + 1);
+
+    if (!possibleEmbed.startsWith("http")) {
+      possibleEmbed = `https://${possibleEmbed}`;
+    }
+
+    return decodeURIComponent(possibleEmbed);
+  });
+
+  return possibleEmbedChoices
+    .slice(Math.max(possibleEmbedChoices.length - maxAmount, 0))
+    .reverse();
+};
+
+const getUrlEmbed = async (url) => {
+  const possibleUrlEmbeds = getPossibleUrlEmbeds(url);
+  for (const possibleUrl of possibleUrlEmbeds) {
+    try {
+      const embeddedUrl = new URL(possibleUrl);
+      await got(embeddedUrl.href, {
+        timeout: 3000,
+        method: "HEAD",
+        responseType: "json",
+        headers: {
+          accept: "*/*",
+        },
+      });
+
+      return embeddedUrl;
+    } catch (error) {
+      // do nothing
+    }
+  }
+
+  return null;
+};
+
 const getLoopControls = ({ limit, offset, length, reverse }) => {
   if (reverse) {
     const startIndex = length - 1 - offset;
@@ -48,6 +120,9 @@ const getLoopControls = ({ limit, offset, length, reverse }) => {
 };
 
 const getItemsToDownload = ({
+  archive,
+  archiveUrl,
+  basePath,
   feed,
   limit,
   offset,
@@ -55,6 +130,8 @@ const getItemsToDownload = ({
   before,
   after,
   episodeRegex,
+  episodeTemplate,
+  includeEpisodeImages,
 }) => {
   const { startIndex, limitCheck, next } = getLoopControls({
     limit,
@@ -66,6 +143,8 @@ const getItemsToDownload = ({
   let i = startIndex;
   const items = [];
 
+  const savedArchive = archive ? getArchive(archive) : [];
+
   while (limitCheck(i)) {
     const { title, pubDate } = feed.items[i];
     const pubDateDay = dayjs(new Date(pubDate));
@@ -98,9 +177,59 @@ const getItemsToDownload = ({
       }
     }
 
+    const { url: episodeAudioUrl, ext: audioFileExt } =
+      getEpisodeAudioUrlAndExt(feed.items[i]);
+    const key = getArchiveKey({
+      prefix: archiveUrl,
+      name: getArchiveFilename({
+        pubDate,
+        name: title,
+        ext: audioFileExt,
+      }),
+    });
+
+    if (key && savedArchive.includes(key)) {
+      isValid = false;
+    }
+
     if (isValid) {
       const item = feed.items[i];
       item._originalIndex = i;
+      item._extra_downloads = [];
+
+      if (includeEpisodeImages) {
+        const episodeImageUrl = getImageUrl(item);
+
+        if (episodeImageUrl) {
+          const episodeImageFileExt = getUrlExt(episodeImageUrl);
+          const episodeImageArchiveKey = getArchiveKey({
+            prefix: archiveUrl,
+            name: getArchiveFilename({
+              pubDate,
+              name: title,
+              ext: episodeImageFileExt,
+            }),
+          });
+
+          if (!savedArchive.includes(episodeImageArchiveKey)) {
+            const episodeImageName = getFilename({
+              item,
+              feed,
+              url: episodeAudioUrl,
+              ext: episodeImageFileExt,
+              template: episodeTemplate,
+            });
+
+            const outputImagePath = path.resolve(basePath, episodeImageName);
+            item._extra_downloads.push({
+              url: episodeImageUrl,
+              outputPath: outputImagePath,
+              key: episodeImageArchiveKey,
+            });
+          }
+        }
+      }
+
       items.push(item);
     }
 
@@ -111,9 +240,9 @@ const getItemsToDownload = ({
 };
 
 const logFeedInfo = (feed) => {
-  console.log(`Title: ${feed.title}`);
-  console.log(`Description: ${feed.description}`);
-  console.log(`Total Episodes: ${feed.items ? feed.items.length : 0}`);
+  logMessage(feed.title);
+  logMessage(feed.description);
+  logMessage();
 };
 
 const ITEM_LIST_FORMATS = {
@@ -148,6 +277,7 @@ const logItemsList = ({
       pubDate: item.pubDate,
     };
   });
+
   if (!tableData.length) {
     logErrorAndExit("No episodes found with provided criteria to list");
   }
@@ -159,46 +289,9 @@ const logItemsList = ({
   }
 };
 
-const logItemInfo = (item, logLevel) => {
-  const { title, pubDate } = item;
-
-  logMessage(`Title: ${title}`, logLevel);
-  logMessage(`Publish Date: ${pubDate}`, logLevel);
-};
-
-const getArchiveKey = ({ prefix, name }) => {
-  return `${prefix}-${name}`;
-};
-
-const writeToArchive = ({ key, archive }) => {
-  let archiveResult = [];
-  const archivePath = path.resolve(process.cwd(), archive);
-
-  if (fs.existsSync(archivePath)) {
-    archiveResult = JSON.parse(fs.readFileSync(archivePath));
-  }
-
-  if (!archiveResult.includes(key)) {
-    archiveResult.push(key);
-  }
-
-  fs.writeFileSync(archivePath, JSON.stringify(archiveResult, null, 4));
-};
-
-const getIsInArchive = ({ key, archive }) => {
-  const archivePath = path.resolve(process.cwd(), archive);
-
-  if (!fs.existsSync(archivePath)) {
-    return false;
-  }
-
-  const archiveResult = JSON.parse(fs.readFileSync(archivePath));
-  return archiveResult.includes(key);
-};
-
 const writeFeedMeta = ({ outputPath, feed, key, archive, override }) => {
   if (key && archive && getIsInArchive({ key, archive })) {
-    logMessage("Feed metadata exists in archive. Skipping write");
+    logMessage("Feed metadata exists in archive. Skipping write...");
     return;
   }
 
@@ -225,24 +318,35 @@ const writeFeedMeta = ({ outputPath, feed, key, archive, override }) => {
         )
       );
     } else {
-      logMessage("Feed metadata exists locally. Skipping write");
+      logMessage("Feed metadata exists locally. Skipping write...");
     }
 
     if (key && archive && !getIsInArchive({ key, archive })) {
       try {
         writeToArchive({ key, archive });
       } catch (error) {
-        logError("Error writing to archive", error);
+        throw new Error(`Error writing to archive: ${error.toString()}`);
       }
     }
   } catch (error) {
-    logError("Unable to save metadata file for episode", error);
+    throw new Error(
+      `Unable to save metadata file for feed: ${error.toString()}`
+    );
   }
 };
 
-const writeItemMeta = ({ outputPath, item, key, archive, override }) => {
+const writeItemMeta = ({
+  marker,
+  outputPath,
+  item,
+  key,
+  archive,
+  override,
+}) => {
   if (key && archive && getIsInArchive({ key, archive })) {
-    logMessage("Episode metadata exists in archive. Skipping write");
+    logMessage(
+      `${marker} | Episode metadata exists in archive. Skipping write...`
+    );
     return;
   }
 
@@ -267,23 +371,25 @@ const writeItemMeta = ({ outputPath, item, key, archive, override }) => {
         )
       );
     } else {
-      logMessage("Episode metadata exists locally. Skipping write");
+      logMessage(
+        `${marker} | Episode metadata exists locally. Skipping write...`
+      );
     }
 
     if (key && archive && !getIsInArchive({ key, archive })) {
       try {
         writeToArchive({ key, archive });
       } catch (error) {
-        logError("Error writing to archive", error);
+        throw new Error("Error writing to archive", error);
       }
     }
   } catch (error) {
-    logError("Unable to save meta file for episode", error);
+    throw new Error("Unable to save meta file for episode", error);
   }
 };
 
 const getUrlExt = (url) => {
-  const { pathname } = _url.parse(url);
+  const { pathname } = new URL(url);
 
   if (!pathname) {
     return "";
@@ -349,149 +455,8 @@ const getImageUrl = ({ image, itunes }) => {
   return null;
 };
 
-const BYTES_IN_MB = 1000000;
-const printProgress = ({ percent, total, transferred }) => {
-  if (!getShouldOutputProgressIndicator()) {
-    /*
-      Non-TTY environments do not have access to `stdout.clearLine` and
-      `stdout.cursorTo`. Skip download progress logging in these environments.
-    */
-    return;
-  }
-
-  let line = "downloading...";
-  const percentRounded = (percent * 100).toFixed(2);
-
-  if (transferred > 0) {
-    /*
-     * Got has a bug where it'll set percent to 1 when the download first starts.
-     * Ignore percent until transfer has started.
-     */
-    line += ` ${percentRounded}%`;
-
-    if (total) {
-      const totalMBs = total / BYTES_IN_MB;
-      const roundedTotalMbs = totalMBs.toFixed(2);
-      line += ` of ${roundedTotalMbs} MB`;
-    }
-  }
-
-  process.stdout.clearLine();
-  process.stdout.cursorTo(0);
-  process.stdout.write(line);
-};
-
-const download = async ({
-  url,
-  outputPath,
-  key,
-  archive,
-  override,
-  onSkip,
-  onBeforeDownload,
-  onAfterDownload,
-}) => {
-  if (key && archive && getIsInArchive({ key, archive })) {
-    if (onSkip) {
-      onSkip();
-    }
-
-    logMessage("Download exists in archive. Skipping");
-    return;
-  }
-
-  if (!override && fs.existsSync(outputPath)) {
-    if (onSkip) {
-      onSkip();
-    }
-
-    logMessage("Download exists locally. Skipping");
-    return;
-  }
-
-  if (onBeforeDownload) {
-    onBeforeDownload();
-  }
-
-  const headResponse = await got(url, {
-    timeout: 5000,
-    method: "HEAD",
-    responseType: "json",
-    headers: {
-      accept: "*/*",
-    },
-  });
-
-  const removeFile = () => {
-    if (fs.existsSync(outputPath)) {
-      fs.unlinkSync(outputPath);
-    }
-  };
-
-  const expectedSize =
-    headResponse &&
-    headResponse.headers &&
-    headResponse.headers["content-length"]
-      ? parseInt(headResponse.headers["content-length"])
-      : 0;
-
-  if (!getShouldOutputProgressIndicator()) {
-    logMessage(
-      `Starting download${
-        expectedSize
-          ? ` of ${(expectedSize / BYTES_IN_MB).toFixed(2)} MB`
-          : "..."
-      }`
-    );
-  }
-
-  try {
-    await pipeline(
-      got.stream(url).on("downloadProgress", (progress) => {
-        printProgress(progress);
-      }),
-      fs.createWriteStream(outputPath)
-    );
-  } catch (error) {
-    removeFile();
-
-    throw error;
-  } finally {
-    if (getShouldOutputProgressIndicator()) {
-      console.log();
-    }
-  }
-
-  const fileSize = fs.statSync(outputPath).size;
-
-  if (fileSize === 0) {
-    removeFile();
-    throw new Error("Unable to write to file. Suggestion: verify permissions");
-  }
-
-  if (expectedSize && !isNaN(expectedSize) && expectedSize !== fileSize) {
-    logMessage(
-      "File size differs from expected content length. Suggestion: verify file works as expected",
-      LOG_LEVELS.important
-    );
-    logMessage(outputPath, LOG_LEVELS.important);
-  }
-
-  if (onAfterDownload) {
-    onAfterDownload();
-  }
-
-  if (key && archive && !getIsInArchive({ key, archive })) {
-    try {
-      writeToArchive({ key, archive });
-    } catch (error) {
-      logError("Error writing to archive", error);
-    }
-  }
-};
-
 const getFeed = async (url) => {
-  const { href } = _url.parse(url);
+  const { href } = new URL(url);
 
   let feed;
   try {
@@ -503,7 +468,7 @@ const getFeed = async (url) => {
   return feed;
 };
 
-const runFfmpeg = ({
+const runFfmpeg = async ({
   feed,
   item,
   itemIndex,
@@ -517,8 +482,7 @@ const runFfmpeg = ({
   }
 
   if (!outputPath.endsWith(".mp3")) {
-    logError("Not an .mp3 file. Unable to run ffmpeg.");
-    return;
+    throw new Error("Not an .mp3 file. Unable to run ffmpeg.");
   }
 
   let command = `ffmpeg -loglevel quiet -i "${outputPath}"`;
@@ -570,26 +534,21 @@ const runFfmpeg = ({
   const tmpMp3Path = `${outputPath}.tmp.mp3`;
   command += ` "${tmpMp3Path}"`;
 
-  logMessage("Running ffmpeg...");
-
   try {
-    execSync(command);
+    await execWithPromise(command, { stdio: "ignore" });
   } catch (error) {
-    logError("Error running ffmpeg", error);
-
     if (fs.existsSync(tmpMp3Path)) {
-      logMessage("Cleaning up temporary file...");
       fs.unlinkSync(tmpMp3Path);
     }
 
-    return;
+    throw error;
   }
 
   fs.unlinkSync(outputPath);
   fs.renameSync(tmpMp3Path, outputPath);
 };
 
-const runExec = ({ exec, outputPodcastPath, episodeFilename }) => {
+const runExec = async ({ exec, outputPodcastPath, episodeFilename }) => {
   const filenameBase = episodeFilename.substring(
     0,
     episodeFilename.lastIndexOf(".")
@@ -597,23 +556,22 @@ const runExec = ({ exec, outputPodcastPath, episodeFilename }) => {
   const execCmd = exec
     .replace(/{}/g, `"${outputPodcastPath}"`)
     .replace(/{filenameBase}/g, `"${filenameBase}"`);
-  try {
-    execSync(execCmd, { stdio: "ignore" });
-  } catch (error) {
-    logError(`--exec process error: exit code ${error.status}`, error);
-  }
+
+  await execWithPromise(execCmd, { stdio: "ignore" });
 };
 
-module.exports = {
-  download,
+export {
+  getArchive,
+  getIsInArchive,
   getArchiveKey,
+  writeToArchive,
   getEpisodeAudioUrlAndExt,
   getFeed,
   getImageUrl,
   getItemsToDownload,
   getUrlExt,
+  getUrlEmbed,
   logFeedInfo,
-  logItemInfo,
   ITEM_LIST_FORMATS,
   logItemsList,
   writeFeedMeta,

package/bin/validate.js

@@ -1,4 +1,6 @@
-const { logErrorAndExit } = require("./logger");
+import { sync as commandExistsSync } from "command-exists";
+
+import { logErrorAndExit } from "./logger.js";
 
 const createParseNumber = ({ min, name, required = true }) => {
   return (value) => {
@@ -24,6 +26,10 @@ const createParseNumber = ({ min, name, required = true }) => {
   };
 };
 
-module.exports = {
-  createParseNumber,
+const hasFfmpeg = () => {
+  if (!commandExistsSync("ffmpeg")) {
+    logErrorAndExit('option specified requires "ffmpeg" be available');
+  }
 };
+
+export { createParseNumber, hasFfmpeg };

package/package.json

@@ -1,12 +1,11 @@
 {
   "name": "podcast-dl",
-  "version": "6.1.0",
+  "version": "7.1.0",
   "description": "A CLI for downloading podcasts.",
+  "type": "module",
   "bin": "./bin/bin.js",
   "scripts": {
-    "build": "rimraf ./binaries && npm run pkg",
     "lint": "eslint ./bin",
-    "pkg": "npx pkg ./ --targets node16-linux-x64,node16-win-x64,node16-macos-x64 --out-path ./binaries",
     "release": "standard-version"
   },
   "lint-staged": {
@@ -27,7 +26,7 @@
     "cli"
   ],
   "engines": {
-    "node": ">=12.16.2"
+    "node": ">=14.17.6"
   },
   "repository": {
     "type": "git",
@@ -48,10 +47,14 @@
     "standard-version": "^9.0.0"
   },
   "dependencies": {
+    "command-exists": "^1.2.9",
     "commander": "^5.1.0",
     "dayjs": "^1.8.25",
     "filenamify": "^4.1.0",
     "got": "^11.0.2",
-    "rss-parser": "^3.7.6"
+    "p-limit": "^4.0.0",
+    "pluralize": "^8.0.0",
+    "rss-parser": "^3.7.6",
+    "throttle-debounce": "^3.0.1"
   }
 }