podcast-dl 7.3.1 → 8.0.0

package/README.md

@@ -10,6 +10,12 @@ A CLI for downloading podcasts with a focus on archiving.
 
 `npx podcast-dl --url <PODCAST_RSS_URL>`
 
+### Binaries
+
+[Visit the releases page](https://github.com/lightpohl/podcast-dl/releases) and download the latest binary for your system.
+
+`podcast-dl --url <PODCAST_RSS_URL>`
+
 ### [More Examples](./docs/examples.md)
 
 ## Options
@@ -38,8 +44,8 @@ Type values surrounded in square brackets (`[]`) can be used as used as boolean
 | --reverse                |                     | false    | Reverse download direction and start at last RSS item.                                                                                                                                                                |
 | --info                   |                     | false    | Print retrieved podcast info instead of downloading.                                                                                                                                                                  |
 | --list                   | [String]            | false    | Print episode list instead of downloading. Defaults to "table" when used as a boolean option. "json" is also supported.                                                                                               |
-| --exec                   | String              | false    | Execute a command after each episode is downloaded.                                                                                                                                                                   |
-| --filter-url-tacking     |                     | false    | Attempts to extract the direct download link of an episode if detected (**experimental**).                                                                                                                            |
+| --exec                   | String              | false    | Execute a command after each episode is downloaded. See "Templating" for more details.                                                                                                                                |
+| --parser-config          | String              | false    | Path to JSON file that will be parsed and used to override the default config passed to [rss-parser](https://github.com/rbren/rss-parser#xml-options).                                                                |
 | --version                |                     | false    | Output the version number.                                                                                                                                                                                            |
 | --help                   |                     | false    | Output usage information.                                                                                                                                                                                             |
 
@@ -50,7 +56,7 @@ Type values surrounded in square brackets (`[]`) can be used as used as boolean
 
 ## Templating
 
-Options that support templating allow users to specify a template for the generated filename(s). The provided template will replace all matched keywords with the related data described below. Each keyword must be wrapped in two braces like so:
+Options that support templating allow users to specify a template for the generated filename(s) or option. The provided template will replace all matched keywords with the related data described below. Each keyword must be wrapped in two braces like so:
 
 `--out-dir "./{{podcast_title}}"`
 
@@ -70,12 +76,12 @@ Options that support templating allow users to specify a template for the genera
 - `podcast_title`: Title of the podcast feed.
 - `podcast_link`: `link` value provided for the podcast feed. Typically the homepage URL.
 
-## Executing Process After Downloading Episode
-
-Option to execute command after downloading episode with `{}` being a placeholder for the downloaded episode and `{filenameBase}` for the filename without extension.
+### `--exec`
 
-- Example to convert all episodes to mp3 with 192k: `ffmpeg -i {} -b:a 192k -f mp3 {filenameBase}.mp3`
-- Example to move all episodes to folder: `mv {} /mnt/media_server/`
+- `episode_path`: The path to the downloaded episode.
+- `episode_path_base`: The path to the folder of the downloaded episode.
+- `episode_filname`: The filename of the episode.
+- `episode_filename_base`: The filename of the episode without its extension.
 
 ## Log Levels
 

package/bin/async.js

@@ -21,7 +21,6 @@ import {
   runExec,
   writeItemMeta,
   writeToArchive,
-  getUrlEmbed,
   getIsInArchive,
 } from "./util.js";
 
@@ -37,7 +36,6 @@ const download = async ({
   archive,
   override,
   onAfterDownload,
-  filterUrlTracking,
 }) => {
   const logMessage = getLogMessageWithMarker(marker);
   if (!override && fs.existsSync(outputPath)) {
@@ -50,18 +48,7 @@ const download = async ({
     return;
   }
 
-  let embeddedUrl = null;
-  if (filterUrlTracking) {
-    logMessage("Attempting to find embedded URL...");
-    embeddedUrl = await getUrlEmbed(url);
-
-    if (!embeddedUrl) {
-      logMessage("Unable to find embedded URL. Defaulting to full address");
-    }
-  }
-
-  const finalUrl = embeddedUrl || url;
-  const headResponse = await got(finalUrl, {
+  const headResponse = await got(url, {
     timeout: 5000,
     method: "HEAD",
     responseType: "json",
@@ -108,7 +95,7 @@ const download = async ({
     });
 
     await pipeline(
-      got.stream(finalUrl).on("downloadProgress", onDownloadProgress),
+      got.stream(url).on("downloadProgress", onDownloadProgress),
       fs.createWriteStream(tempOutputPath)
     );
   } catch (error) {
@@ -155,7 +142,6 @@ let downloadItemsAsync = async ({
   episodeTemplate,
   exec,
   feed,
-  filterUrlTracking,
   includeEpisodeMeta,
   mono,
   override,
@@ -193,7 +179,6 @@ let downloadItemsAsync = async ({
         archive,
         override,
         marker,
-        filterUrlTracking,
         key: getArchiveKey({
           prefix: archiveUrl,
           name: getArchiveFilename({
@@ -219,7 +204,12 @@ let downloadItemsAsync = async ({
 
           if (exec) {
             logMessage("Running exec...");
-            await runExec({ exec, outputPodcastPath, episodeFilename });
+            await runExec({
+              exec,
+              basePath,
+              outputPodcastPath,
+              episodeFilename,
+            });
           }
 
           numEpisodesDownloaded += 1;

package/bin/bin.js

@@ -3,9 +3,9 @@
 import fs from "fs";
 import _path from "path";
 import commander from "commander";
-import { createRequire } from "module";
 import pluralize from "pluralize";
 
+import { setupCommander } from "./commander.js";
 import { download } from "./async.js";
 import {
   getArchiveKey,
@@ -16,9 +16,7 @@ import {
   logFeedInfo,
   logItemsList,
   writeFeedMeta,
-  ITEM_LIST_FORMATS,
 } from "./util.js";
-import { createParseNumber, hasFfmpeg } from "./validate.js";
 import {
   ERROR_STATUSES,
   LOG_LEVELS,
@@ -29,100 +27,7 @@ import {
 import { getFolderName, getSafeName } from "./naming.js";
 import { downloadItemsAsync } from "./async.js";
 
-const require = createRequire(import.meta.url);
-const { version } = require("../package.json");
-
-commander
-  .version(version)
-  .option("--url <string>", "url to podcast rss feed")
-  .option("--out-dir <path>", "specify output directory", "./{{podcast_title}}")
-  .option(
-    "--archive [path]",
-    "download or write only items not listed in archive file"
-  )
-  .option(
-    "--episode-template <string>",
-    "template for generating episode related filenames",
-    "{{release_date}}-{{title}}"
-  )
-  .option("--include-meta", "write out podcast metadata to json")
-  .option(
-    "--include-episode-meta",
-    "write out individual episode metadata to json"
-  )
-  .option("--include-episode-images", "download found episode images")
-  .option(
-    "--offset <number>",
-    "offset episode to start downloading from (most recent = 0)",
-    createParseNumber({ min: 0, name: "--offset" }),
-    0
-  )
-  .option(
-    "--limit <number>",
-    "max amount of episodes to download",
-    createParseNumber({ min: 1, name: "--limit", require: false })
-  )
-  .option(
-    "--episode-regex <string>",
-    "match episode title against regex before downloading"
-  )
-  .option(
-    "--after <string>",
-    "download episodes only after this date (inclusive)"
-  )
-  .option(
-    "--before <string>",
-    "download episodes only before this date (inclusive)"
-  )
-  .option(
-    "--add-mp3-metadata",
-    "attempts to add a base level of metadata to .mp3 files using ffmpeg",
-    hasFfmpeg
-  )
-  .option(
-    "--adjust-bitrate <string>",
-    "attempts to adjust bitrate of .mp3 files using ffmpeg",
-    hasFfmpeg
-  )
-  .option(
-    "--mono",
-    "attempts to force .mp3 files into mono using ffmpeg",
-    hasFfmpeg
-  )
-  .option("--override", "override local files on collision")
-  .option("--reverse", "download episodes in reverse order")
-  .option("--info", "print retrieved podcast info instead of downloading")
-  .option(
-    "--list [table|json]",
-    "print episode info instead of downloading",
-    (value) => {
-      if (
-        value !== ITEM_LIST_FORMATS.table &&
-        value !== ITEM_LIST_FORMATS.json
-      ) {
-        logErrorAndExit(
-          `${value} is an invalid format for --list\nUse "table" or "json"`
-        );
-      }
-
-      return value;
-    }
-  )
-  .option(
-    "--exec <string>",
-    "Execute a command after each episode is downloaded"
-  )
-  .option(
-    "--threads <number>",
-    "the number of downloads that can happen concurrently",
-    createParseNumber({ min: 1, max: 32, name: "threads" }),
-    1
-  )
-  .option(
-    "--filter-url-tracking",
-    "attempts to extract the direct download link of an episode if detected (experimental)"
-  )
-  .parse(process.argv);
+setupCommander(commander, process.argv);
 
 const {
   url,
@@ -143,7 +48,7 @@ const {
   exec,
   mono,
   threads,
-  filterUrlTracking,
+  parserConfig,
   addMp3Metadata: addMp3MetadataFlag,
   adjustBitrate: bitrate,
 } = commander;
@@ -157,13 +62,15 @@ const main = async () => {
 
   const { hostname, pathname } = new URL(url);
   const archiveUrl = `${hostname}${pathname}`;
-  const feed = await getFeed(url);
+  const feed = await getFeed(url, parserConfig);
   const basePath = _path.resolve(
     process.cwd(),
     getFolderName({ feed, template: outDir })
   );
 
-  logFeedInfo(feed);
+  if (info) {
+    logFeedInfo(feed);
+  }
 
   if (list) {
     if (feed.items && feed.items.length) {
@@ -187,6 +94,8 @@ const main = async () => {
     process.exit(0);
   }
 
+  logFeedInfo(feed);
+
   if (!fs.existsSync(basePath)) {
     logMessage(`${basePath} does not exist. Creating...`, LOG_LEVELS.important);
     fs.mkdirSync(basePath, { recursive: true });
@@ -290,7 +199,6 @@ const main = async () => {
     override,
     targetItems,
     threads,
-    filterUrlTracking,
   });
 
   if (hasErrors && numEpisodesDownloaded !== targetItems.length) {

package/bin/commander.js

@@ -0,0 +1,100 @@
+import { ITEM_LIST_FORMATS } from "./util.js";
+import { createParseNumber, hasFfmpeg } from "./validate.js";
+import { logErrorAndExit } from "./logger.js";
+
+export const setupCommander = (commander, argv) => {
+  commander
+    .version("8.0.0")
+    .option("--url <string>", "url to podcast rss feed")
+    .option(
+      "--out-dir <path>",
+      "specify output directory",
+      "./{{podcast_title}}"
+    )
+    .option(
+      "--archive [path]",
+      "download or write only items not listed in archive file"
+    )
+    .option(
+      "--episode-template <string>",
+      "template for generating episode related filenames",
+      "{{release_date}}-{{title}}"
+    )
+    .option("--include-meta", "write out podcast metadata to json")
+    .option(
+      "--include-episode-meta",
+      "write out individual episode metadata to json"
+    )
+    .option("--include-episode-images", "download found episode images")
+    .option(
+      "--offset <number>",
+      "offset episode to start downloading from (most recent = 0)",
+      createParseNumber({ min: 0, name: "--offset" }),
+      0
+    )
+    .option(
+      "--limit <number>",
+      "max amount of episodes to download",
+      createParseNumber({ min: 1, name: "--limit", require: false })
+    )
+    .option(
+      "--episode-regex <string>",
+      "match episode title against regex before downloading"
+    )
+    .option(
+      "--after <string>",
+      "download episodes only after this date (inclusive)"
+    )
+    .option(
+      "--before <string>",
+      "download episodes only before this date (inclusive)"
+    )
+    .option(
+      "--add-mp3-metadata",
+      "attempts to add a base level of metadata to .mp3 files using ffmpeg",
+      hasFfmpeg
+    )
+    .option(
+      "--adjust-bitrate <string>",
+      "attempts to adjust bitrate of .mp3 files using ffmpeg",
+      hasFfmpeg
+    )
+    .option(
+      "--mono",
+      "attempts to force .mp3 files into mono using ffmpeg",
+      hasFfmpeg
+    )
+    .option("--override", "override local files on collision")
+    .option("--reverse", "download episodes in reverse order")
+    .option("--info", "print retrieved podcast info instead of downloading")
+    .option(
+      "--list [table|json]",
+      "print episode info instead of downloading",
+      (value) => {
+        if (!ITEM_LIST_FORMATS.includes(value)) {
+          logErrorAndExit(
+            `${value} is an invalid format for --list\nUse one of the following: ${ITEM_LIST_FORMATS.join(
+              ", "
+            )}`
+          );
+        }
+
+        return value;
+      }
+    )
+    .option(
+      "--exec <string>",
+      "Execute a command after each episode is downloaded"
+    )
+    .option(
+      "--threads <number>",
+      "the number of downloads that can happen concurrently",
+      createParseNumber({ min: 1, max: 32, name: "threads" }),
+      1
+    )
+    .option(
+      "--parser-config <string>",
+      "path to JSON config to override RSS parser"
+    )
+    .parse(argv);
+};

package/bin/naming.js

@@ -41,8 +41,8 @@ const getFilename = ({ item, ext, url, feed, template }) => {
 
 const getFolderName = ({ feed, template }) => {
   const templateReplacementsTuples = [
-    ["podcast_title", feed.title ? getSafeName(feed.title) : ""],
-    ["podcast_link", feed.link ? getSafeName(feed.link) : ""],
+    ["podcast_title", feed.title || ""],
+    ["podcast_link", feed.link || ""],
   ];
 
   let name = template;
@@ -55,7 +55,7 @@ const getFolderName = ({ feed, template }) => {
       : name.replace(replaceRegex, "");
   });
 
-  return name;
+  return getSafeName(name);
 };
 
 const getArchiveFilename = ({ pubDate, name, ext }) => {

package/bin/util.js

@@ -2,7 +2,6 @@ import rssParser from "rss-parser";
 import path from "path";
 import fs from "fs";
 import dayjs from "dayjs";
-import got from "got";
 import util from "util";
 import { exec } from "child_process";
 
@@ -11,10 +10,6 @@ import { getArchiveFilename, getFilename } from "./naming.js";
 
 const execWithPromise = util.promisify(exec);
 
-const parser = new rssParser({
-  defaultRSS: 2.0,
-});
-
 const getTempPath = (path) => {
   return `${path}.tmp`;
 };
@@ -23,14 +18,30 @@ const getArchiveKey = ({ prefix, name }) => {
   return `${prefix}-${name}`;
 };
 
-const getArchive = (archive) => {
-  const archivePath = path.resolve(process.cwd(), archive);
+const getPublicObject = (object) => {
+  const output = {};
+  Object.keys(object).forEach((key) => {
+    if (!key.startsWith("_") && object[key]) {
+      output[key] = object[key];
+    }
+  });
+
+  return output;
+};
 
-  if (!fs.existsSync(archivePath)) {
-    return [];
+const getJsonFile = (filePath) => {
+  const fullPath = path.resolve(process.cwd(), filePath);
+
+  if (!fs.existsSync(fullPath)) {
+    return null;
   }
 
-  return JSON.parse(fs.readFileSync(archivePath));
+  return JSON.parse(fs.readFileSync(fullPath));
+};
+
+const getArchive = (archive) => {
+  const archiveContent = getJsonFile(archive);
+  return archiveContent === null ? [] : archiveContent;
 };
 
 const writeToArchive = ({ key, archive }) => {
@@ -49,76 +60,28 @@ const getIsInArchive = ({ key, archive }) => {
   return archiveResult.includes(key);
 };
 
-const getPossibleUrlEmbeds = (url, maxAmount = 5) => {
-  const fullUrl = new URL(url);
-  const possibleStartIndexes = [];
-
-  for (let i = 0; i < fullUrl.pathname.length; i++) {
-    if (fullUrl.pathname[i] === "/") {
-      possibleStartIndexes.push(i);
-    }
-  }
-
-  const possibleEmbedChoices = possibleStartIndexes.map((startIndex) => {
-    let possibleEmbed = fullUrl.pathname.slice(startIndex + 1);
-
-    if (!possibleEmbed.startsWith("http")) {
-      possibleEmbed = `https://${possibleEmbed}`;
-    }
-
-    return decodeURIComponent(possibleEmbed);
-  });
-
-  return possibleEmbedChoices
-    .slice(Math.max(possibleEmbedChoices.length - maxAmount, 0))
-    .reverse();
-};
-
-const getUrlEmbed = async (url) => {
-  const possibleUrlEmbeds = getPossibleUrlEmbeds(url);
-  for (const possibleUrl of possibleUrlEmbeds) {
-    try {
-      const embeddedUrl = new URL(possibleUrl);
-      await got(embeddedUrl.href, {
-        timeout: 3000,
-        method: "HEAD",
-        responseType: "json",
-        headers: {
-          accept: "*/*",
-        },
-      });
-
-      return embeddedUrl;
-    } catch (error) {
-      // do nothing
-    }
-  }
-
-  return null;
-};
-
-const getLoopControls = ({ limit, offset, length, reverse }) => {
+const getLoopControls = ({ offset, length, reverse }) => {
   if (reverse) {
     const startIndex = length - 1 - offset;
-    const min = limit ? Math.max(startIndex - limit, -1) : -1;
-    const limitCheck = (i) => i > min;
+    const min = -1;
+    const shouldGo = (i) => i > min;
     const decrement = (i) => i - 1;
 
     return {
       startIndex,
-      limitCheck,
+      shouldGo,
       next: decrement,
     };
   }
 
   const startIndex = 0 + offset;
-  const max = limit ? Math.min(startIndex + limit, length) : length;
-  const limitCheck = (i) => i < max;
+  const max = length;
+  const shouldGo = (i) => i < max;
   const increment = (i) => i + 1;
 
   return {
     startIndex,
-    limitCheck,
+    shouldGo,
     next: increment,
   };
 };
@@ -137,8 +100,7 @@ const getItemsToDownload = ({
   episodeTemplate,
   includeEpisodeImages,
 }) => {
-  const { startIndex, limitCheck, next } = getLoopControls({
-    limit,
+  const { startIndex, shouldGo, next } = getLoopControls({
     offset,
     reverse,
     length: feed.items.length,
@@ -149,7 +111,7 @@ const getItemsToDownload = ({
 
   const savedArchive = archive ? getArchive(archive) : [];
 
-  while (limitCheck(i)) {
+  while (shouldGo(i)) {
     const { title, pubDate } = feed.items[i];
     const pubDateDay = dayjs(new Date(pubDate));
     let isValid = true;
@@ -238,7 +200,7 @@ const getItemsToDownload = ({
     i = next(i);
   }
 
-  return items;
+  return limit ? items.slice(0, limit) : items;
 };
 
 const logFeedInfo = (feed) => {
@@ -247,10 +209,7 @@ const logFeedInfo = (feed) => {
   logMessage();
 };
 
-const ITEM_LIST_FORMATS = {
-  table: "table",
-  json: "json",
-};
+const ITEM_LIST_FORMATS = ["table", "json"];
 
 const logItemsList = ({
   type,
@@ -272,23 +231,28 @@ const logItemsList = ({
     episodeRegex,
   });
 
-  const tableData = items.map((item) => {
-    return {
+  if (!items.length) {
+    logErrorAndExit("No episodes found with provided criteria to list");
+  }
+
+  const isJson = type === "json";
+
+  const output = items.map((item) => {
+    const data = {
       episodeNum: feed.items.length - item._originalIndex,
       title: item.title,
       pubDate: item.pubDate,
     };
+
+    return data;
   });
 
-  if (!tableData.length) {
-    logErrorAndExit("No episodes found with provided criteria to list");
+  if (isJson) {
+    console.log(JSON.stringify(output));
+    return;
   }
 
-  if (type === ITEM_LIST_FORMATS.json) {
-    console.log(JSON.stringify(tableData));
-  } else {
-    console.table(tableData);
-  }
+  console.table(output);
 };
 
 const writeFeedMeta = ({ outputPath, feed, key, archive, override }) => {
@@ -296,15 +260,7 @@ const writeFeedMeta = ({ outputPath, feed, key, archive, override }) => {
     logMessage("Feed metadata exists in archive. Skipping...");
     return;
   }
-
-  const output = {};
-  ["title", "description", "link", "feedUrl", "managingEditor"].forEach(
-    (key) => {
-      if (feed[key]) {
-        output[key] = feed[key];
-      }
-    }
-  );
+  const output = getPublicObject(feed);
 
   try {
     if (override || !fs.existsSync(outputPath)) {
@@ -340,12 +296,7 @@ const writeItemMeta = ({
     return;
   }
 
-  const output = {};
-  ["title", "contentSnippet", "pubDate", "creator"].forEach((key) => {
-    if (item[key]) {
-      output[key] = item[key];
-    }
-  });
+  const output = getPublicObject(item);
 
   try {
     if (override || !fs.existsSync(outputPath)) {
@@ -433,7 +384,19 @@ const getImageUrl = ({ image, itunes }) => {
   return null;
 };
 
-const getFeed = async (url) => {
+const getFeed = async (url, parserConfig) => {
+  const defaultConfig = {
+    defaultRSS: 2.0,
+  };
+
+  const config = parserConfig ? getJsonFile(parserConfig) : defaultConfig;
+
+  if (parserConfig && !config) {
+    logErrorAndExit(`Unable to load parser config: ${parserConfig}`);
+  }
+
+  const parser = new rssParser(config);
+
   const { href } = new URL(url);
 
   let feed;
@@ -526,14 +489,22 @@ const runFfmpeg = async ({
   fs.renameSync(tmpMp3Path, outputPath);
 };
 
-const runExec = async ({ exec, outputPodcastPath, episodeFilename }) => {
-  const filenameBase = episodeFilename.substring(
+const runExec = async ({
+  exec,
+  basePath,
+  outputPodcastPath,
+  episodeFilename,
+}) => {
+  const episodeFilenameBase = episodeFilename.substring(
     0,
     episodeFilename.lastIndexOf(".")
   );
+
   const execCmd = exec
-    .replace(/{}/g, `"${outputPodcastPath}"`)
-    .replace(/{filenameBase}/g, `"${filenameBase}"`);
+    .replace(/{{episode_path}}/g, `"${outputPodcastPath}"`)
+    .replace(/{{episode_path_base}}/g, `"${basePath}"`)
+    .replace(/{{episode_filename}}/g, `"${episodeFilename}"`)
+    .replace(/{{episode_filename_base}}/g, `"${episodeFilenameBase}"`);
 
   await execWithPromise(execCmd, { stdio: "ignore" });
 };
@@ -549,7 +520,6 @@ export {
   getItemsToDownload,
   getTempPath,
   getUrlExt,
-  getUrlEmbed,
   logFeedInfo,
   ITEM_LIST_FORMATS,
   logItemsList,

package/package.json

@@ -1,11 +1,13 @@
 {
   "name": "podcast-dl",
-  "version": "7.3.1",
+  "version": "8.0.0",
   "description": "A CLI for downloading podcasts.",
   "type": "module",
   "bin": "./bin/bin.js",
   "scripts": {
+    "build": "rimraf ./binaries && npx webpack && npm run pkg",
     "lint": "eslint ./bin",
+    "pkg": "npx pkg ./dist/podcast-dl.js --targets node14-linux-x64,node14-win-x64,node14-macos-x64 --out-path ./binaries",
     "release": "standard-version"
   },
   "lint-staged": {
@@ -44,7 +46,8 @@
     "lint-staged": "^10.1.7",
     "prettier": "2.3.2",
     "rimraf": "^3.0.2",
-    "standard-version": "^9.0.0"
+    "standard-version": "^9.0.0",
+    "webpack-cli": "^5.0.1"
   },
   "dependencies": {
     "command-exists": "^1.2.9",
@@ -54,7 +57,7 @@
     "got": "^11.0.2",
     "p-limit": "^4.0.0",
     "pluralize": "^8.0.0",
-    "rss-parser": "^3.7.6",
+    "rss-parser": "^3.12.0",
     "throttle-debounce": "^3.0.1"
   }
 }