podcast-dl 7.0.0 → 7.3.0

package/README.md

@@ -1,8 +1,6 @@
 # podcast-dl
 
-## A CLI for downloading podcasts with a focus on archiving.
-
-![podcast-dl example gif](./docs/podcast-dl-example.gif)
+A CLI for downloading podcasts with a focus on archiving.
 
 ## How to Use
 
@@ -12,6 +10,8 @@
 
 `npx podcast-dl --url <PODCAST_RSS_URL>`
 
+### [More Examples](./docs/examples.md)
+
 ## Options
 
 Type values surrounded in square brackets (`[]`) can be used as used as boolean options (no argument required).
@@ -20,6 +20,7 @@ Type values surrounded in square brackets (`[]`) can be used as used as boolean
 | ------------------------ | ------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | --url                    | String              | true     | URL to podcast RSS feed.                                                                                                                                                                                              |
 | --out-dir                | String              | false    | Specify output directory for episodes and metadata. Defaults to "./{{podcast_title}}". See "Templating" for more details.                                                                                             |
+| --threads                | Number              | false    | Determines the number of downloads that will happen concurrently. Default is 1.                                                                                                                                       |
 | --archive                | [String]            | false    | Download or write out items not listed in archive file. Generates archive file at path if not found. Defaults to "./{{podcast_title}}/archive.json" when used as a boolean option. See "Templating" for more details. |
 | --episode-template       | String              | false    | Template for generating episode related filenames. See "Templating" for details.                                                                                                                                      |
 | --include-meta           |                     | false    | Write out podcast metadata to JSON.                                                                                                                                                                                   |
@@ -38,7 +39,7 @@ Type values surrounded in square brackets (`[]`) can be used as used as boolean
 | --info                   |                     | false    | Print retrieved podcast info instead of downloading.                                                                                                                                                                  |
 | --list                   | [String]            | false    | Print episode list instead of downloading. Defaults to "table" when used as a boolean option. "json" is also supported.                                                                                               |
 | --exec                   | String              | false    | Execute a command after each episode is downloaded.                                                                                                                                                                   |
-| --threads                | Number              | false    | Determines the number of downloads that will happen concurrently. Default is 1.                                                                                                                                       |
+| --filter-url-tacking     |                     | false    | Attempts to extract the direct download link of an episode if detected (**experimental**).                                                                                                                            |
 | --version                |                     | false    | Output the version number.                                                                                                                                                                                            |
 | --help                   |                     | false    | Output usage information.                                                                                                                                                                                             |
 

package/bin/async.js

@@ -16,10 +16,12 @@ import { getArchiveFilename, getFilename } from "./naming.js";
 import {
   getEpisodeAudioUrlAndExt,
   getArchiveKey,
+  getTempPath,
   runFfmpeg,
   runExec,
   writeItemMeta,
   writeToArchive,
+  getUrlEmbed,
 } from "./util.js";
 
 const pipeline = promisify(stream.pipeline);
@@ -34,6 +36,7 @@ const download = async ({
   archive,
   override,
   onAfterDownload,
+  filterUrlTracking,
 }) => {
   const logMessage = getLogMessageWithMarker(marker);
   if (!override && fs.existsSync(outputPath)) {
@@ -41,7 +44,18 @@ const download = async ({
     return;
   }
 
-  const headResponse = await got(url, {
+  let embeddedUrl = null;
+  if (filterUrlTracking) {
+    logMessage("Attempting to find embedded URL...");
+    embeddedUrl = await getUrlEmbed(url);
+
+    if (!embeddedUrl) {
+      logMessage("Unable to find embedded URL. Defaulting to full address");
+    }
+  }
+
+  const finalUrl = embeddedUrl || url;
+  const headResponse = await got(finalUrl, {
     timeout: 5000,
     method: "HEAD",
     responseType: "json",
@@ -50,9 +64,10 @@ const download = async ({
     },
   });
 
+  const tempOutputPath = getTempPath(outputPath);
   const removeFile = () => {
-    if (fs.existsSync(outputPath)) {
-      fs.unlinkSync(outputPath);
+    if (fs.existsSync(tempOutputPath)) {
+      fs.unlinkSync(tempOutputPath);
     }
   };
 
@@ -87,15 +102,15 @@ const download = async ({
     });
 
     await pipeline(
-      got.stream(url).on("downloadProgress", onDownloadProgress),
-      fs.createWriteStream(outputPath)
+      got.stream(finalUrl).on("downloadProgress", onDownloadProgress),
+      fs.createWriteStream(tempOutputPath)
     );
   } catch (error) {
     removeFile();
     throw error;
   }
 
-  const fileSize = fs.statSync(outputPath).size;
+  const fileSize = fs.statSync(tempOutputPath).size;
 
   if (fileSize === 0) {
     removeFile();
@@ -108,13 +123,7 @@ const download = async ({
     return;
   }
 
-  if (expectedSize && !isNaN(expectedSize) && expectedSize !== fileSize) {
-    logMessage(
-      "File size differs from expected content length. Suggestion: verify file works as expected",
-      LOG_LEVELS.important
-    );
-    logMessage(`${outputPath}`, LOG_LEVELS.important);
-  }
+  fs.renameSync(tempOutputPath, outputPath);
 
   logMessage("Download complete!");
 
@@ -140,6 +149,7 @@ let downloadItemsAsync = async ({
   episodeTemplate,
   exec,
   feed,
+  filterUrlTracking,
   includeEpisodeMeta,
   mono,
   override,
@@ -177,6 +187,7 @@ let downloadItemsAsync = async ({
         archive,
         override,
         marker,
+        filterUrlTracking,
         key: getArchiveKey({
           prefix: archiveUrl,
           name: getArchiveFilename({

package/bin/bin.js

@@ -2,7 +2,6 @@
 
 import fs from "fs";
 import _path from "path";
-import _url from "url";
 import commander from "commander";
 import { createRequire } from "module";
 import pluralize from "pluralize";
@@ -119,6 +118,10 @@ commander
     createParseNumber({ min: 1, max: 32, name: "threads" }),
     1
   )
+  .option(
+    "--filter-url-tracking",
+    "attempts to extract the direct download link of an episode if detected (experimental)"
+  )
   .parse(process.argv);
 
 const {
@@ -140,6 +143,7 @@ const {
   exec,
   mono,
   threads,
+  filterUrlTracking,
   addMp3Metadata: addMp3MetadataFlag,
   adjustBitrate: bitrate,
 } = commander;
@@ -151,7 +155,7 @@ const main = async () => {
     logErrorAndExit("No URL provided");
   }
 
-  const { hostname, pathname } = _url.parse(url);
+  const { hostname, pathname } = new URL(url);
   const archiveUrl = `${hostname}${pathname}`;
   const feed = await getFeed(url);
   const basePath = _path.resolve(
@@ -286,8 +290,27 @@ const main = async () => {
     override,
     targetItems,
     threads,
+    filterUrlTracking,
   });
 
+  if (hasErrors && numEpisodesDownloaded !== targetItems.length) {
+    logMessage(
+      `\n${numEpisodesDownloaded} of ${pluralize(
+        "episode",
+        targetItems.length,
+        true
+      )} downloaded\n`
+    );
+  } else {
+    logMessage(
+      `\nSuccessfully downloaded ${pluralize(
+        "episode",
+        numEpisodesDownloaded,
+        true
+      )}\n`
+    );
+  }
+
   if (numEpisodesDownloaded === 0) {
     process.exit(ERROR_STATUSES.nothingDownloaded);
   }

package/bin/util.js

@@ -1,8 +1,8 @@
-import _url from "url";
 import rssParser from "rss-parser";
 import path from "path";
 import fs from "fs";
 import dayjs from "dayjs";
+import got from "got";
 import util from "util";
 import { exec } from "child_process";
 
@@ -15,6 +15,10 @@ const parser = new rssParser({
   defaultRSS: 2.0,
 });
 
+const getTempPath = (path) => {
+  return `${path}.tmp`;
+};
+
 const getArchiveKey = ({ prefix, name }) => {
   return `${prefix}-${name}`;
 };
@@ -45,6 +49,54 @@ const getIsInArchive = ({ key, archive }) => {
   return archiveResult.includes(key);
 };
 
+const getPossibleUrlEmbeds = (url, maxAmount = 5) => {
+  const fullUrl = new URL(url);
+  const possibleStartIndexes = [];
+
+  for (let i = 0; i < fullUrl.pathname.length; i++) {
+    if (fullUrl.pathname[i] === "/") {
+      possibleStartIndexes.push(i);
+    }
+  }
+
+  const possibleEmbedChoices = possibleStartIndexes.map((startIndex) => {
+    let possibleEmbed = fullUrl.pathname.slice(startIndex + 1);
+
+    if (!possibleEmbed.startsWith("http")) {
+      possibleEmbed = `https://${possibleEmbed}`;
+    }
+
+    return decodeURIComponent(possibleEmbed);
+  });
+
+  return possibleEmbedChoices
+    .slice(Math.max(possibleEmbedChoices.length - maxAmount, 0))
+    .reverse();
+};
+
+const getUrlEmbed = async (url) => {
+  const possibleUrlEmbeds = getPossibleUrlEmbeds(url);
+  for (const possibleUrl of possibleUrlEmbeds) {
+    try {
+      const embeddedUrl = new URL(possibleUrl);
+      await got(embeddedUrl.href, {
+        timeout: 3000,
+        method: "HEAD",
+        responseType: "json",
+        headers: {
+          accept: "*/*",
+        },
+      });
+
+      return embeddedUrl;
+    } catch (error) {
+      // do nothing
+    }
+  }
+
+  return null;
+};
+
 const getLoopControls = ({ limit, offset, length, reverse }) => {
   if (reverse) {
     const startIndex = length - 1 - offset;
@@ -247,28 +299,18 @@ const writeFeedMeta = ({ outputPath, feed, key, archive, override }) => {
     return;
   }
 
-  const title = feed.title || null;
-  const description = feed.description || null;
-  const link = feed.link || null;
-  const feedUrl = feed.feedUrl || null;
-  const managingEditor = feed.managingEditor || null;
+  const output = {};
+  ["title", "description", "link", "feedUrl", "managingEditor"].forEach(
+    (key) => {
+      if (feed[key]) {
+        output[key] = feed[key];
+      }
+    }
+  );
 
   try {
     if (override || !fs.existsSync(outputPath)) {
-      fs.writeFileSync(
-        outputPath,
-        JSON.stringify(
-          {
-            title,
-            description,
-            link,
-            feedUrl,
-            managingEditor,
-          },
-          null,
-          4
-        )
-      );
+      fs.writeFileSync(outputPath, JSON.stringify(output, null, 4));
     } else {
       logMessage("Feed metadata exists locally. Skipping write...");
     }
@@ -302,26 +344,16 @@ const writeItemMeta = ({
     return;
   }
 
-  const title = item.title || null;
-  const descriptionText = item.contentSnippet || null;
-  const pubDate = item.pubDate || null;
-  const creator = item.creator || null;
+  const output = {};
+  ["title", "contentSnippet", "pubDate", "creator"].forEach((key) => {
+    if (item[key]) {
+      output[key] = item[key];
+    }
+  });
 
   try {
     if (override || !fs.existsSync(outputPath)) {
-      fs.writeFileSync(
-        outputPath,
-        JSON.stringify(
-          {
-            title,
-            pubDate,
-            creator,
-            descriptionText,
-          },
-          null,
-          4
-        )
-      );
+      fs.writeFileSync(outputPath, JSON.stringify(output, null, 4));
     } else {
       logMessage(
         `${marker} | Episode metadata exists locally. Skipping write...`
@@ -341,7 +373,7 @@ const writeItemMeta = ({
 };
 
 const getUrlExt = (url) => {
-  const { pathname } = _url.parse(url);
+  const { pathname } = new URL(url);
 
   if (!pathname) {
     return "";
@@ -408,7 +440,7 @@ const getImageUrl = ({ image, itunes }) => {
 };
 
 const getFeed = async (url) => {
-  const { href } = _url.parse(url);
+  const { href } = new URL(url);
 
   let feed;
   try {
@@ -521,7 +553,9 @@ export {
   getFeed,
   getImageUrl,
   getItemsToDownload,
+  getTempPath,
   getUrlExt,
+  getUrlEmbed,
   logFeedInfo,
   ITEM_LIST_FORMATS,
   logItemsList,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "podcast-dl",
-  "version": "7.0.0",
+  "version": "7.3.0",
   "description": "A CLI for downloading podcasts.",
   "type": "module",
   "bin": "./bin/bin.js",