podverse-parser 5.1.12-alpha.0 → 5.1.12-alpha.1

package/dist/config/index.d.ts

@@ -14,5 +14,8 @@ export declare const config: {
         baseUrl: string;
         secretKey: string;
     };
+    parser: {
+        addRemoteItemsToMQ: boolean;
+    };
 };
 //# sourceMappingURL=index.d.ts.map

package/dist/config/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;CAgBlB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;CAmBlB,CAAC"}

package/dist/config/index.js

@@ -16,5 +16,8 @@ exports.config = {
         authKey: process.env.PODCAST_INDEX_AUTH_KEY || '',
         baseUrl: process.env.PODCAST_INDEX_BASE_URL || '',
         secretKey: process.env.PODCAST_INDEX_SECRET_KEY || ''
+    },
+    parser: {
+        addRemoteItemsToMQ: process.env.PARSER_ADD_REMOTE_ITEMS_TO_MQ === 'true',
     }
 };

package/dist/lib/rss/item/itemValue.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"itemValue.d.ts","sourceRoot":"","sources":["../../../../src/lib/rss/item/itemValue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,IAAI,EAG1B,OAAO,EACR,MAAM,cAAc,CAAC;AAGtB,eAAO,MAAM,qBAAqB,GAChC,YAAY,OAAO,EACnB,MAAM,IAAI,EACV,SAAS,OAAO,EAChB,6BAA6B,aAAa,kBA2D3C,CAAC"}
+{"version":3,"file":"itemValue.d.ts","sourceRoot":"","sources":["../../../../src/lib/rss/item/itemValue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,IAAI,EAG1B,OAAO,EACR,MAAM,cAAc,CAAC;AAGtB,eAAO,MAAM,qBAAqB,GAChC,YAAY,OAAO,EACnB,MAAM,IAAI,EACV,SAAS,OAAO,EAChB,6BAA6B,aAAa,kBA8D3C,CAAC"}

package/dist/lib/rss/item/itemValue.js

@@ -10,6 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.handleParsedItemValue = void 0;
+const podverse_helpers_1 = require("podverse-helpers");
 const podverse_orm_1 = require("podverse-orm");
 const item_1 = require("@parser/lib/compat/partytime/item");
 const handleParsedItemValue = (parsedItem, item, channel, transactionalEntityManager) => __awaiter(void 0, void 0, void 0, function* () {
@@ -53,6 +54,9 @@ const handleParsedItemValue = (parsedItem, item, channel, transactionalEntityMan
                     }
                     const itemValueTimeSplitRemoteItemDto = itemValueTimeSplitDto.item_value_time_splits_remote_item;
                     if (itemValueTimeSplitRemoteItemDto) {
+                        if (!(0, podverse_helpers_1.hasValidFeedUuid)(itemValueTimeSplitRemoteItemDto)) {
+                            continue;
+                        }
                         yield itemValueTimeSplitRemoteItemService.update(item_value_time_split, itemValueTimeSplitRemoteItemDto);
                     }
                     else {

package/dist/lib/rss/parser.d.ts

@@ -10,5 +10,11 @@ export type ParseRSSFeedAndSaveToDatabaseOptions = {
     forceParse: boolean;
     onDemandParserEvent: ParseRSSOnDemandParserEvent;
 };
-export declare const parseRSSFeedAndSaveToDatabase: (url: string, podcast_index_id: number, options: ParseRSSFeedAndSaveToDatabaseOptions) => Promise<void>;
+export declare const parseRSSFeedAndSaveToDatabase: (url: string, podcast_index_id: number, options: ParseRSSFeedAndSaveToDatabaseOptions) => Promise<{
+    remoteItemsToParse: {
+        url: string;
+        podcast_index_id: number;
+        options: ParseRSSFeedAndSaveToDatabaseOptions;
+    }[];
+}>;
 //# sourceMappingURL=parser.d.ts.map

package/dist/lib/rss/parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../../src/lib/rss/parser.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EAIxB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAa,MAAM,oBAAoB,CAAC;AAgC3D,eAAO,MAAM,kBAAkB,GAAU,KAAK,MAAM,wBAUnD,CAAC;AAQF,MAAM,MAAM,2BAA2B,GAAG;IACxC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,0BAA0B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,IAAI,EAAE,uBAAuB,GAAG,IAAI,CAAC;CACtC,CAAA;AAED,MAAM,MAAM,oCAAoC,GAAG;IACjD,UAAU,EAAE,OAAO,CAAC;IACpB,mBAAmB,EAAE,2BAA2B,CAAC;CAClD,CAAA;AAED,eAAO,MAAM,6BAA6B,GACxC,KAAK,MAAM,EACX,kBAAkB,MAAM,EACxB,SAAS,oCAAoC,kBAuI9C,CAAC"}
+{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../../src/lib/rss/parser.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EAKxB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAa,MAAM,oBAAoB,CAAC;AAiC3D,eAAO,MAAM,kBAAkB,GAAU,KAAK,MAAM,wBAUnD,CAAC;AAQF,MAAM,MAAM,2BAA2B,GAAG;IACxC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,0BAA0B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,IAAI,EAAE,uBAAuB,GAAG,IAAI,CAAC;CACtC,CAAA;AAED,MAAM,MAAM,oCAAoC,GAAG;IACjD,UAAU,EAAE,OAAO,CAAC;IACpB,mBAAmB,EAAE,2BAA2B,CAAC;CAClD,CAAA;AAgBD,eAAO,MAAM,6BAA6B,GACxC,KAAK,MAAM,EACX,kBAAkB,MAAM,EACxB,SAAS,oCAAoC;;;;;;EA4I9C,CAAC"}

package/dist/lib/rss/parser.js

@@ -27,6 +27,7 @@ const loggerService_1 = require("@parser/factories/loggerService");
 // import { NotificationsServiceFactory } from '@parser/factories/notificationsService';
 const _request_1 = require("../_request");
 const parsedFeed_1 = require("./hash/parsedFeed");
+const config_1 = require("@parser/config");
 /*
   NOTE: All RSS feeds that have a podcast_index_id will be saved to the database.
   RSS feeds without podcast_index_id (Add By RSS feeds) will NOT be saved to the database.
@@ -41,6 +42,20 @@ const getAndParseRSSFeed = (url) => __awaiter(void 0, void 0, void 0, function*
     return parsedFeed;
 });
 exports.getAndParseRSSFeed = getAndParseRSSFeed;
+// Handle request delay for specific domains to avoid rate limiting
+function handleRateLimitRequestDelay(url) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const delayConfig = [
+            { regex: /^https?:\/\/(www\.)?wavlake\.com/, delay: 5000 },
+        ];
+        for (const { regex, delay } of delayConfig) {
+            if (regex.test(url)) {
+                yield (0, podverse_helpers_1.sleep)(delay);
+                break;
+            }
+        }
+    });
+}
 const parseRSSFeedAndSaveToDatabase = (url, podcast_index_id, options) => __awaiter(void 0, void 0, void 0, function* () {
     const { onDemandParserEvent } = options;
     const onDemandParserEventService = new podverse_orm_1.OnDemandParserEventService();
@@ -71,6 +86,7 @@ const parseRSSFeedAndSaveToDatabase = (url, podcast_index_id, options) => __awai
         if (!url || !podcast_index_id) {
             throw new Error(`parseRSSFeedAndSaveToDatabase: url or podcast_index_id is missing for ${url} ${podcast_index_id}`);
         }
+        yield handleRateLimitRequestDelay(url);
         loggerService_1.loggerService.info(`parseRSSFeedAndSaveToDatabase url: ${url} podcast_index_id: ${podcast_index_id}`);
         feed = yield (0, feed_1.handleGetRSSFeed)(url, podcast_index_id);
         if (!(0, podverse_orm_1.checkIfFeedFlagStatusShouldParse)(feed.feed_flag_status.id)) {
@@ -151,12 +167,15 @@ const parseRSSFeedAndSaveToDatabase = (url, podcast_index_id, options) => __awai
             }
         }
     }
-    if (channel) {
-        yield (0, remoteItemParser_1.handleAllRemoteItemsFeedParsing)(channel, {
-            accountId: (onDemandParserEvent === null || onDemandParserEvent === void 0 ? void 0 : onDemandParserEvent.accountId) || null,
-            remoteParentPodcastIndexId: podcast_index_id,
-        });
+    if (config_1.config.parser.addRemoteItemsToMQ) {
+        if (channel) {
+            const remoteItems = yield (0, remoteItemParser_1.handleAllRemoteItemsFeedParsing)(channel, {
+                accountId: (onDemandParserEvent === null || onDemandParserEvent === void 0 ? void 0 : onDemandParserEvent.accountId) || null,
+                remoteParentPodcastIndexId: podcast_index_id,
+            });
+            return { remoteItemsToParse: remoteItems };
+        }
     }
-    return;
+    return { remoteItemsToParse: [] };
 });
 exports.parseRSSFeedAndSaveToDatabase = parseRSSFeedAndSaveToDatabase;

package/dist/lib/rss/remoteItemParser.d.ts

@@ -1,7 +1,14 @@
 import { Channel } from 'podverse-orm';
+import type { ParseRSSFeedAndSaveToDatabaseOptions } from '@parser/lib/rss/parser';
 export type OnDemandParserRemoteItemParams = {
     accountId: number | null;
     remoteParentPodcastIndexId: number;
 };
-export declare const handleAllRemoteItemsFeedParsing: (channel: Channel, params: OnDemandParserRemoteItemParams) => Promise<void>;
+type RemoteItemsQueueMessage = {
+    url: string;
+    podcast_index_id: number;
+    options: ParseRSSFeedAndSaveToDatabaseOptions;
+};
+export declare const handleAllRemoteItemsFeedParsing: (channel: Channel, params: OnDemandParserRemoteItemParams) => Promise<RemoteItemsQueueMessage[]>;
+export {};
 //# sourceMappingURL=remoteItemParser.d.ts.map

package/dist/lib/rss/remoteItemParser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"remoteItemParser.d.ts","sourceRoot":"","sources":["../../../src/lib/rss/remoteItemParser.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,OAAO,EAUR,MAAM,cAAc,CAAC;AAUtB,MAAM,MAAM,8BAA8B,GAAG;IAC3C,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,0BAA0B,EAAE,MAAM,CAAC;CACpC,CAAA;AAoED,eAAO,MAAM,+BAA+B,GAAU,SAAS,OAAO,EAAE,QAAQ,8BAA8B,kBAO7G,CAAC"}
+{"version":3,"file":"remoteItemParser.d.ts","sourceRoot":"","sources":["../../../src/lib/rss/remoteItemParser.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,OAAO,EAUR,MAAM,cAAc,CAAC;AAEtB,OAAO,KAAK,EAAE,oCAAoC,EAAE,MAAM,wBAAwB,CAAC;AAQnF,MAAM,MAAM,8BAA8B,GAAG;IAC3C,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,0BAA0B,EAAE,MAAM,CAAC;CACpC,CAAA;AAED,KAAK,uBAAuB,GAAG;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,oCAAoC,CAAC;CAC/C,CAAA;AA4DD,eAAO,MAAM,+BAA+B,GAAU,SAAS,OAAO,EAAE,QAAQ,8BAA8B,KAAG,OAAO,CAAC,uBAAuB,EAAE,CAWjJ,CAAC"}

package/dist/lib/rss/remoteItemParser.js

@@ -13,21 +13,7 @@ exports.handleAllRemoteItemsFeedParsing = void 0;
 const podverse_helpers_1 = require("podverse-helpers");
 const podverse_orm_1 = require("podverse-orm");
 const podcastIndex_1 = require("@parser/factories/podcastIndex");
-const parser_1 = require("@parser/lib/rss/parser");
 const loggerService_1 = require("@parser/factories/loggerService");
-function handleRequestDelay(url) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const delayConfig = [
-            { regex: /^https?:\/\/(www\.)?wavlake\.com/, delay: 5000 },
-        ];
-        for (const { regex, delay } of delayConfig) {
-            if (regex.test(url)) {
-                yield (0, podverse_helpers_1.sleep)(delay);
-                break;
-            }
-        }
-    });
-}
 const handleRemoteItemsFeedParsing = (feedGuidsToParse, params) => __awaiter(void 0, void 0, void 0, function* () {
     var _a, _b;
     const { accountId, remoteParentPodcastIndexId } = params;
@@ -53,33 +39,41 @@ const handleRemoteItemsFeedParsing = (feedGuidsToParse, params) => __awaiter(voi
             }
         }
     }
+    const queueMessages = [];
     for (const piFeedData of piFeedDatas) {
         const feedService = new podverse_orm_1.FeedService();
-        let feed = yield feedService.getByUrlAndPodcastIndexId({
+        const feed = yield feedService.getByUrlAndPodcastIndexId({
             url: piFeedData.url,
             podcast_index_id: piFeedData.id
         });
         if (!feed) {
-            yield handleRequestDelay(piFeedData.url);
             loggerService_1.loggerService.info(`handleRemoteItemsFeedParsing: ${piFeedData.url} ${piFeedData.id}`);
-            yield (0, parser_1.parseRSSFeedAndSaveToDatabase)(piFeedData.url, piFeedData.id, {
-                forceParse: false,
-                onDemandParserEvent: {
-                    accountId,
-                    remoteParentPodcastIndexId,
-                    type: podverse_helpers_1.OnDemandParserEventType.REMOTE_ITEM,
-                },
+            queueMessages.push({
+                url: piFeedData.url,
+                podcast_index_id: piFeedData.id,
+                options: {
+                    forceParse: false,
+                    onDemandParserEvent: {
+                        accountId,
+                        remoteParentPodcastIndexId,
+                        type: podverse_helpers_1.OnDemandParserEventType.REMOTE_ITEM,
+                    }
+                }
             });
         }
     }
+    return queueMessages;
 });
 const handleAllRemoteItemsFeedParsing = (channel, params) => __awaiter(void 0, void 0, void 0, function* () {
     const channelService = new podverse_orm_1.ChannelService();
     const latestChannel = yield channelService.get(channel.id);
-    yield handleRemoteItemsPodrollParsing(channel, params);
-    yield handleRemoteItemsPublisherParsing(channel, params);
-    yield handleRemoteItemsChannelParsing(channel, params);
-    yield handleRemoteItemsItemValueTimeSplitParsing(latestChannel, params);
+    const results = [];
+    const podrollResults = yield handleRemoteItemsPodrollParsing(latestChannel, params);
+    const publisherResults = yield handleRemoteItemsPublisherParsing(latestChannel, params);
+    const channelResults = yield handleRemoteItemsChannelParsing(latestChannel, params);
+    const timeSplitResults = yield handleRemoteItemsItemValueTimeSplitParsing(latestChannel, params);
+    results.push(...podrollResults, ...publisherResults, ...channelResults, ...timeSplitResults);
+    return results;
 });
 exports.handleAllRemoteItemsFeedParsing = handleAllRemoteItemsFeedParsing;
 const handleRemoteItemsPodrollParsing = (channel, params) => __awaiter(void 0, void 0, void 0, function* () {
@@ -89,8 +83,9 @@ const handleRemoteItemsPodrollParsing = (channel, params) => __awaiter(void 0, v
         const channelPodrollRemoteItemService = new podverse_orm_1.ChannelPodrollRemoteItemService();
         const channelPodrollRemoteItems = yield channelPodrollRemoteItemService.getAll(channelPodroll);
         const feedGuidsToParse = channelPodrollRemoteItems.map(remoteItem => remoteItem.feed_guid);
-        yield handleRemoteItemsFeedParsing(feedGuidsToParse, params);
+        return yield handleRemoteItemsFeedParsing(feedGuidsToParse, params);
     }
+    return [];
 });
 const handleRemoteItemsPublisherParsing = (channel, params) => __awaiter(void 0, void 0, void 0, function* () {
     const channelPublisherService = new podverse_orm_1.ChannelPublisherService();
@@ -99,14 +94,15 @@ const handleRemoteItemsPublisherParsing = (channel, params) => __awaiter(void 0,
         const channelPublisherRemoteItemService = new podverse_orm_1.ChannelPublisherRemoteItemService();
         const channelPublisherRemoteItems = yield channelPublisherRemoteItemService.getAll(channelPublisher);
         const feedGuidsToParse = channelPublisherRemoteItems.map(remoteItem => remoteItem.feed_guid);
-        yield handleRemoteItemsFeedParsing(feedGuidsToParse, params);
+        return yield handleRemoteItemsFeedParsing(feedGuidsToParse, params);
     }
+    return [];
 });
 const handleRemoteItemsChannelParsing = (channel, params) => __awaiter(void 0, void 0, void 0, function* () {
     const channelRemoteItemService = new podverse_orm_1.ChannelRemoteItemService();
     const channelRemoteItems = yield channelRemoteItemService.getAll(channel);
     const feedGuidsToParse = channelRemoteItems.map(remoteItem => remoteItem.feed_guid);
-    yield handleRemoteItemsFeedParsing(feedGuidsToParse, params);
+    return yield handleRemoteItemsFeedParsing(feedGuidsToParse, params);
 });
 const handleRemoteItemsItemValueTimeSplitParsing = (channel, params) => __awaiter(void 0, void 0, void 0, function* () {
     var _a, _b;
@@ -120,6 +116,7 @@ const handleRemoteItemsItemValueTimeSplitParsing = (channel, params) => __awaite
                 'item_values.item_value_time_splits.item_value_time_split_remote_item'
             ]
         });
+        const results = [];
         for (const item of items) {
             if (item) {
                 if (((_a = item.item_values) === null || _a === void 0 ? void 0 : _a.length) > 0) {
@@ -128,7 +125,10 @@ const handleRemoteItemsItemValueTimeSplitParsing = (channel, params) => __awaite
                             for (const itemValueTimeSplit of itemValue.item_value_time_splits) {
                                 if (itemValueTimeSplit.item_value_time_split_remote_item) {
                                     const feedGuid = itemValueTimeSplit.item_value_time_split_remote_item.feed_guid;
-                                    yield handleRemoteItemsFeedParsing([feedGuid], params);
+                                    const res = yield handleRemoteItemsFeedParsing([feedGuid], params);
+                                    if (res && res.length > 0) {
+                                        results.push(...res);
+                                    }
                                 }
                             }
                         }
@@ -136,5 +136,7 @@ const handleRemoteItemsItemValueTimeSplitParsing = (channel, params) => __awaite
                 }
             }
         }
+        return results;
     }
+    return [];
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "podverse-parser",
-  "version": "5.1.12-alpha.0",
+  "version": "5.1.12-alpha.1",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -17,9 +17,9 @@
   "license": "AGPLv3",
   "dependencies": {
     "module-alias": "^2.2.3",
-    "podverse-external-services": "^5.1.12-alpha.0",
-    "podverse-helpers": "^5.1.12-alpha.0",
-    "podverse-orm": "^5.1.12-alpha.0",
+    "podverse-external-services": "^5.1.12-alpha.2",
+    "podverse-helpers": "^5.1.12-alpha.1",
+    "podverse-orm": "^5.1.12-alpha.1",
     "podverse-partytime": "^5.0.0"
   },
   "devDependencies": {