podverse-external-services 5.1.1-alpha.1 → 5.1.1-alpha.11

package/dist/config/index.d.ts

@@ -11,8 +11,5 @@ export declare const config: {
         clientSecret: string;
         mode: string;
     };
-    podcastIndex: {
-        recentlyUpdatedDataInterval: number;
-    };
 };
 //# sourceMappingURL=index.d.ts.map

package/dist/config/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;CAgBlB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,MAAM;;;;;;;;;;;;;CAalB,CAAA"}

package/dist/config/index.js

@@ -13,8 +13,5 @@ exports.config = {
         clientId: process.env.PAYPAL_CLIENT_ID || '',
         clientSecret: process.env.PAYPAL_CLIENT_SECRET || '',
         mode: process.env.PAYPAL_MODE || 'sandbox'
-    },
-    podcastIndex: {
-        recentlyUpdatedDataInterval: parseInt(process.env.PODCAST_INDEX_RECENTLY_UPDATED_DATA_INTERVAL || '1800', 10)
     }
 };

package/dist/services/podcast-index/index.d.ts

@@ -10,11 +10,20 @@ export declare class PodcastIndexService {
     secretKey: string;
     constructor({ authKey, baseUrl, secretKey }: Constructor);
     podcastIndexAPIRequest: (url: string, config?: any) => Promise<any>;
-    getRecentlyUpdatedData: () => Promise<any[]>;
-    getPodcastByGuid: (podcastGuid: string) => Promise<PodcastByGuidResponse | null>;
-    getValueTagEnabledPodcastIdsRecursively: (accumulatedPodcastIndexIds: number[], startAt?: number) => Promise<number[]>;
-    getValueTagEnabledPodcastIds: () => Promise<number[]>;
-    downloadAndExtractCSV: () => Promise<any[]>;
+    deadFeedsDownloadAndExtractCSV: (resolveHandler: (row: string[]) => void) => Promise<void>;
+    deadFeedsExtractRow: (row: string[]) => {
+        id_to_archive: number;
+        duplicate_id_to_keep: number | null;
+    };
+    podcastGetById: (podcastIndexId: number) => Promise<any | null>;
+    podcastGetByGuid: (podcastGuid: string) => Promise<PodcastByGuidResponse | null>;
+    recentGetData: (sinceRange: number) => Promise<any[]>;
+    trendingGetPodcasts: (max?: number, since?: number, lang?: string, cat?: string) => Promise<{
+        feeds: any[];
+        nextSince?: number;
+    }>;
+    valueGetByPodcastIds: () => Promise<number[]>;
+    valueGetByPodcastIdsRecursively: (accumulatedPodcastIndexIds: number[], startAt?: number) => Promise<number[]>;
 }
 export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/services/podcast-index/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/podcast-index/index.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAG9D,KAAK,WAAW,GAAG;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;CAClB,CAAA;AASD,qBAAa,mBAAmB;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;gBAEZ,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,WAAW;IAMzD,sBAAsB,GAAU,KAAK,MAAM,EAAE,SAAS,GAAG,kBAcxD;IAED,sBAAsB,uBA6BrB;IAED,gBAAgB,GAAU,aAAa,MAAM,KAAG,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAYpF;IAED,uCAAuC,GACrC,4BAA4B,MAAM,EAAE,EAAE,gBAAW,KAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAatE;IAED,4BAA4B,QAAa,OAAO,CAAC,MAAM,EAAE,CAAC,CAMzD;IAED,qBAAqB,QAAa,OAAO,CAAC,GAAG,EAAE,CAAC,CAwC/C;CACF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/podcast-index/index.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAG9D,KAAK,WAAW,GAAG;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;CAClB,CAAA;AASD,qBAAa,mBAAmB;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;gBAEZ,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,WAAW;IAQzD,sBAAsB,GAAU,KAAK,MAAM,EAAE,SAAS,GAAG,kBA+BxD;IAID,8BAA8B,GAAU,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,IAAI,KAAG,OAAO,CAAC,IAAI,CAAC,CAsC9F;IAED,mBAAmB,GAAI,KAAK,MAAM,EAAE;;;MAOnC;IAID,cAAc,GAAU,gBAAgB,MAAM,KAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAQnE;IAED,gBAAgB,GAAU,aAAa,MAAM,KAAG,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAYpF;IAID,aAAa,GAAU,YAAY,MAAM,oBA4BxC;IAID,mBAAmB,GACjB,MAAK,MAAW,EAChB,QAAQ,MAAM,EACd,OAAO,MAAM,EACb,MAAM,MAAM,KACX,OAAO,CAAC;QAAE,KAAK,EAAE,GAAG,EAAE,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAoB/C;IAID,oBAAoB,QAAa,OAAO,CAAC,MAAM,EAAE,CAAC,CAMjD;IAED,+BAA+B,GAC7B,4BAA4B,MAAM,EAAE,EAAE,gBAAW,KAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAatE;CACF"}

package/dist/services/podcast-index/index.js

@@ -19,7 +19,6 @@ const csv_parser_1 = __importDefault(require("csv-parser"));
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const podverse_helpers_1 = require("podverse-helpers");
-const config_1 = require("@external-services/config");
 /*
   NOTE!!!
   The episodeGuid needs to be encoded both on the client-side and server side if it is an http url guid.
@@ -28,19 +27,104 @@ const config_1 = require("@external-services/config");
 */
 class PodcastIndexService {
     constructor({ authKey, baseUrl, secretKey }) {
+        // Request handler
         this.podcastIndexAPIRequest = (url, config) => __awaiter(this, void 0, void 0, function* () {
-            const apiHeaderTime = new Date().getTime() / 1000;
+            var _a, _b, _c;
+            const apiHeaderTime = Math.floor(Date.now() / 1000);
             const hash = (0, sha1_1.default)(this.authKey + this.secretKey + apiHeaderTime).toString(enc_hex_1.default);
-            return (0, podverse_helpers_1.request)(url, Object.assign({ headers: {
-                    'X-Auth-Key': this.authKey,
-                    'X-Auth-Date': apiHeaderTime,
-                    Authorization: hash
-                } }, config));
+            const shouldPreventHeaders = (config === null || config === void 0 ? void 0 : config.preventHeaders) || false;
+            config === null || config === void 0 ? true : delete config.preventHeaders;
+            try {
+                const response = yield (0, podverse_helpers_1.request)(url, Object.assign(Object.assign({}, (shouldPreventHeaders ? {} : {
+                    headers: {
+                        'X-Auth-Key': this.authKey,
+                        'X-Auth-Date': apiHeaderTime,
+                        Authorization: hash
+                    }
+                })), config));
+                return response;
+            }
+            catch (error) {
+                podverse_helpers_1.logger.error('[PodcastIndex] Request failed', {
+                    url,
+                    errorMessage: error === null || error === void 0 ? void 0 : error.message,
+                    errorStack: error === null || error === void 0 ? void 0 : error.stack,
+                    errorResponse: (_a = error === null || error === void 0 ? void 0 : error.response) === null || _a === void 0 ? void 0 : _a.data,
+                    errorStatus: (_b = error === null || error === void 0 ? void 0 : error.response) === null || _b === void 0 ? void 0 : _b.status,
+                    errorHeaders: (_c = error === null || error === void 0 ? void 0 : error.response) === null || _c === void 0 ? void 0 : _c.headers
+                });
+                throw error;
+            }
+        });
+        // Dead Feeds
+        this.deadFeedsDownloadAndExtractCSV = (resolveHandler) => __awaiter(this, void 0, void 0, function* () {
+            const url = 'https://public.podcastindex.org/podcastindex_dead_feeds.csv';
+            const tmpDir = path_1.default.join(__dirname, 'tmp');
+            const filePath = path_1.default.join(tmpDir, 'podcastindex_dead_feeds.csv');
+            if (!fs_1.default.existsSync(tmpDir)) {
+                fs_1.default.mkdirSync(tmpDir);
+            }
+            const data = yield this.podcastIndexAPIRequest(url, { preventHeaders: true, responseType: 'stream' });
+            const writer = fs_1.default.createWriteStream(filePath);
+            data.pipe(writer);
+            yield new Promise((resolve, reject) => {
+                writer.on('finish', () => resolve());
+                writer.on('error', reject);
+            });
+            yield new Promise((resolve, reject) => {
+                const stream = fs_1.default.createReadStream(filePath)
+                    .pipe((0, csv_parser_1.default)({ headers: false, skipLines: 0 }));
+                stream.on('data', (row) => __awaiter(this, void 0, void 0, function* () {
+                    stream.pause();
+                    try {
+                        yield resolveHandler(row);
+                        stream.resume();
+                    }
+                    catch (err) {
+                        stream.destroy(err instanceof Error ? err : new Error(String(err)));
+                    }
+                }));
+                stream.on('end', () => resolve());
+                stream.on('error', reject);
+                stream.on('close', () => resolve());
+            });
+            fs_1.default.unlinkSync(filePath);
+        });
+        this.deadFeedsExtractRow = (row) => {
+            const id_to_archive = row[0];
+            const duplicate_id_to_keep = row[1] || null;
+            return {
+                id_to_archive: parseInt(id_to_archive, 10),
+                duplicate_id_to_keep: duplicate_id_to_keep ? parseInt(duplicate_id_to_keep, 10) : null
+            };
+        };
+        // Podcast
+        this.podcastGetById = (podcastIndexId) => __awaiter(this, void 0, void 0, function* () {
+            const url = `${this.baseUrl}/podcasts/byfeedid?id=${podcastIndexId}`;
+            try {
+                const response = yield this.podcastIndexAPIRequest(url);
+                return response || null;
+            }
+            catch (error) {
+                return null;
+            }
+        });
+        this.podcastGetByGuid = (podcastGuid) => __awaiter(this, void 0, void 0, function* () {
+            const url = `${this.baseUrl}/podcasts/byguid?guid=${podcastGuid}`;
+            let podcastIndexPodcast = null;
+            try {
+                const data = yield this.podcastIndexAPIRequest(url);
+                podcastIndexPodcast = data;
+            }
+            catch (error) {
+                // assume a 404
+            }
+            return podcastIndexPodcast || null;
         });
-        this.getRecentlyUpdatedData = () => __awaiter(this, void 0, void 0, function* () {
-            podverse_helpers_1.logger.info('getRecentlyUpdatedData beginning...');
+        // Recent
+        this.recentGetData = (sinceRange) => __awaiter(this, void 0, void 0, function* () {
+            podverse_helpers_1.logger.info('recentGetData beginning...');
             const currentTimeInSeconds = Math.floor(Date.now() / 1000);
-            const sinceRange = config_1.config.podcastIndex.recentlyUpdatedDataInterval;
             const sinceTimeInSeconds = currentTimeInSeconds - sinceRange;
             const fetchData = (since_1, ...args_1) => __awaiter(this, [since_1, ...args_1], void 0, function* (since, allData = []) {
                 podverse_helpers_1.logger.info(`fetchData since: ${since}, allData.length: ${allData.length}`);
@@ -62,67 +146,44 @@ class PodcastIndexService {
             });
             return fetchData(sinceTimeInSeconds);
         });
-        this.getPodcastByGuid = (podcastGuid) => __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.baseUrl}/podcasts/byguid?guid=${podcastGuid}`;
-            let podcastIndexPodcast = null;
-            try {
-                const data = yield this.podcastIndexAPIRequest(url);
-                podcastIndexPodcast = data;
+        // Trending
+        this.trendingGetPodcasts = (...args_1) => __awaiter(this, [...args_1], void 0, function* (max = 25, since, lang, cat) {
+            const safeMax = Math.min(max, 1000);
+            let url = `${this.baseUrl}/podcasts/trending?max=${safeMax}`;
+            if (since) {
+                url += `&since=${since}`;
             }
-            catch (error) {
-                // assume a 404
+            if (lang) {
+                url += `&lang=${encodeURIComponent(lang)}`;
             }
-            return podcastIndexPodcast || null;
+            if (cat) {
+                url += `&cat=${encodeURIComponent(cat)}`;
+            }
+            podverse_helpers_1.logger.info(`[PodcastIndex] Fetching trending feeds (max: ${safeMax}, since: ${since}, lang: ${lang}, cat: ${cat})`);
+            const response = yield this.podcastIndexAPIRequest(url);
+            return {
+                feeds: response.feeds || [],
+                nextSince: response.nextSince
+            };
         });
-        this.getValueTagEnabledPodcastIdsRecursively = (accumulatedPodcastIndexIds_1, ...args_1) => __awaiter(this, [accumulatedPodcastIndexIds_1, ...args_1], void 0, function* (accumulatedPodcastIndexIds, startAt = 1) {
+        // Value
+        this.valueGetByPodcastIds = () => __awaiter(this, void 0, void 0, function* () {
+            const accumulatedPodcastIndexIds = [];
+            const nextStartAt = 1;
+            const podcastIndexIds = yield this.valueGetByPodcastIdsRecursively(accumulatedPodcastIndexIds, nextStartAt);
+            return podcastIndexIds;
+        });
+        this.valueGetByPodcastIdsRecursively = (accumulatedPodcastIndexIds_1, ...args_1) => __awaiter(this, [accumulatedPodcastIndexIds_1, ...args_1], void 0, function* (accumulatedPodcastIndexIds, startAt = 1) {
             const url = `${this.baseUrl}/podcasts/bytag?podcast-valueTimeSplit=true&max=5000&start_at=${startAt}`;
             const data = yield this.podcastIndexAPIRequest(url);
             for (const feed of data.feeds) {
                 accumulatedPodcastIndexIds.push(feed.id);
             }
             if (data.nextStartAt) {
-                return yield this.getValueTagEnabledPodcastIdsRecursively(accumulatedPodcastIndexIds, data.nextStartAt);
+                return yield this.valueGetByPodcastIdsRecursively(accumulatedPodcastIndexIds, data.nextStartAt);
             }
             return accumulatedPodcastIndexIds;
         });
-        this.getValueTagEnabledPodcastIds = () => __awaiter(this, void 0, void 0, function* () {
-            const accumulatedPodcastIndexIds = [];
-            const nextStartAt = 1;
-            const podcastIndexIds = yield this.getValueTagEnabledPodcastIdsRecursively(accumulatedPodcastIndexIds, nextStartAt);
-            return podcastIndexIds;
-        });
-        this.downloadAndExtractCSV = () => __awaiter(this, void 0, void 0, function* () {
-            const url = 'https://public.podcastindex.org/podcastindex_dead_feeds.csv';
-            const tmpDir = path_1.default.join(__dirname, 'tmp');
-            const filePath = path_1.default.join(tmpDir, 'podcastindex_dead_feeds.csv');
-            if (!fs_1.default.existsSync(tmpDir)) {
-                fs_1.default.mkdirSync(tmpDir);
-            }
-            const data = yield this.podcastIndexAPIRequest(url, { responseType: 'stream' });
-            const writer = fs_1.default.createWriteStream(filePath);
-            data.pipe(writer);
-            yield new Promise((resolve, reject) => {
-                writer.on('finish', () => resolve());
-                writer.on('error', reject);
-            });
-            const results = [];
-            yield new Promise((resolve, reject) => {
-                fs_1.default.createReadStream(filePath)
-                    .pipe((0, csv_parser_1.default)())
-                    .on('data', (data) => results.push(data))
-                    .on('end', resolve)
-                    .on('error', reject);
-            });
-            fs_1.default.unlinkSync(filePath);
-            const parsedResults = results.map((row) => {
-                const [id, duplicateOf] = Object.values(row).map((value) => value.trim());
-                return {
-                    podcast_index_id: parseInt(id, 10),
-                    duplicateOf: duplicateOf ? parseInt(duplicateOf, 10) : null
-                };
-            });
-            return parsedResults;
-        });
         this.authKey = authKey;
         this.baseUrl = baseUrl;
         this.secretKey = secretKey;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "podverse-external-services",
-  "version": "5.1.1-alpha.1",
+  "version": "5.1.1-alpha.11",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -18,7 +18,7 @@
   "devDependencies": {
     "@types/crypto-js": "^4.2.1",
     "@types/http-errors": "^2.0.4",
-    "@types/node": "^20.10.4",
+    "@types/node": "^22.0.0",
     "@types/web-push": "^3.3.2",
     "@typescript-eslint/eslint-plugin": "^6.13.0",
     "@typescript-eslint/parser": "^6.13.0",
@@ -34,7 +34,7 @@
     "module-alias": "^2.2.3",
     "paypal-rest-sdk": "2.0.0-rc.2",
     "podcast-partytime": "^4.8.3",
-    "podverse-helpers": "^5.1.2-alpha.2",
+    "podverse-helpers": "^5.1.2-alpha.5",
     "web-push": "^3.6.3"
   }
 }