podverse-parser 5.0.7 → 5.0.9

package/dist/lib/compat/partytime/item.js

@@ -7,7 +7,7 @@ const value_1 = require("./value");
 const compatItemDto = (parsedItem) => {
     var _a, _b;
     return ({
-        guid: ((_a = parsedItem.guid) === null || _a === void 0 ? void 0 : _a.slice(0, podverse_helpers_1.DATABASE_CONSTANTS.varchar_guid)) || null,
+        guid: ((_a = parsedItem.guid) === null || _a === void 0 ? void 0 : _a.slice(0, podverse_helpers_1.DATABASE_CONSTANTS.varchar_url)) || null,
         guid_enclosure_url: parsedItem.enclosure.url.slice(0, podverse_helpers_1.DATABASE_CONSTANTS.varchar_url),
         pubdate: parsedItem.pubDate || null,
         title: ((_b = parsedItem.title) === null || _b === void 0 ? void 0 : _b.slice(0, podverse_helpers_1.DATABASE_CONSTANTS.varchar_normal)) || null

package/dist/lib/rss/item/item.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"item.d.ts","sourceRoot":"","sources":["../../../../src/lib/rss/item/item.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5C,OAAO,EAAiB,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAe,MAAM,cAAc,CAAC;AAmBtG,eAAO,MAAM,iBAAiB,gBAAuB,OAAO,EAAE,WAAW,OAAO,sBAAsB,kBAAkB,kBAuBvH,CAAC;AAEF,eAAO,MAAM,gBAAgB,eACf,OAAO,WACV,OAAO,sBACI,kBAAkB,8BACV,aAAa,kBAkC1C,CAAC"}
+{"version":3,"file":"item.d.ts","sourceRoot":"","sources":["../../../../src/lib/rss/item/item.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5C,OAAO,EAAiB,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAe,MAAM,cAAc,CAAC;AA+BtG,eAAO,MAAM,iBAAiB,gBAAuB,OAAO,EAAE,WAAW,OAAO,sBAAsB,kBAAkB,kBAyBvH,CAAC;AAEF,eAAO,MAAM,gBAAgB,eACf,OAAO,WACV,OAAO,sBACI,kBAAkB,8BACV,aAAa,kBAkC1C,CAAC"}

package/dist/lib/rss/item/item.js

@@ -29,12 +29,24 @@ const itemTranscript_1 = require("@parser/lib/rss/item/itemTranscript");
 const itemTxt_1 = require("@parser/lib/rss/item/itemTxt");
 const itemValue_1 = require("@parser/lib/rss/item/itemValue");
 const itemChat_1 = require("@parser/lib/rss/item/itemChat");
+const removeDuplicates = (parsedItems) => {
+    const seen = new Set();
+    return parsedItems.reduce((acc, item) => {
+        const enclosureUrl = item.enclosure.url.slice(0, podverse_helpers_1.DATABASE_CONSTANTS.varchar_url);
+        if (!seen.has(enclosureUrl)) {
+            seen.add(enclosureUrl);
+            acc.push(item);
+        }
+        return acc;
+    }, []);
+};
 const handleParsedItems = (parsedItems, channel, channelSeasonIndex) => __awaiter(void 0, void 0, void 0, function* () {
     const itemService = new podverse_orm_1.ItemService();
     const existingItems = yield itemService.getAllItemsByChannel(channel, { select: ['id'] });
     const existingItemIds = existingItems.map(item => item.id);
     const updatedItemIds = [];
-    const parsedItemBatchs = (0, podverse_helpers_1.chunkArray)(parsedItems, 50);
+    const uniqueParsedItems = removeDuplicates(parsedItems);
+    const parsedItemBatchs = (0, podverse_helpers_1.chunkArray)(uniqueParsedItems, 50);
     for (const parsedItemBatch of parsedItemBatchs) {
         yield podverse_orm_1.AppDataSource.manager.transaction((transactionalEntityManager) => __awaiter(void 0, void 0, void 0, function* () {
             for (const parsedItem of parsedItemBatch) {

package/dist/lib/rss/parser.d.ts

@@ -1,4 +1,4 @@
-export declare const parseAllRSSFeeds: () => Promise<any>;
+export declare const parseAllRSSFeeds: () => Promise<void>;
 export declare const getAndParseRSSFeed: (url: string) => Promise<import("podcast-partytime").FeedObject>;
-export declare const parseRSSFeedAndSaveToDatabase: (url: string, podcast_index_id: number) => Promise<any>;
+export declare const parseRSSFeedAndSaveToDatabase: (url: string, podcast_index_id: number) => Promise<void>;
 //# sourceMappingURL=parser.d.ts.map

package/dist/lib/rss/parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../../src/lib/rss/parser.ts"],"names":[],"mappings":"AAeA,eAAO,MAAM,gBAAgB,oBAM5B,CAAC;AAEF,eAAO,MAAM,kBAAkB,QAAe,MAAM,oDASnD,CAAC;AAQF,eAAO,MAAM,6BAA6B,QAAe,MAAM,oBAAoB,MAAM,iBAgExF,CAAC"}
+{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../../src/lib/rss/parser.ts"],"names":[],"mappings":"AAeA,eAAO,MAAM,gBAAgB,qBAM5B,CAAC;AAEF,eAAO,MAAM,kBAAkB,QAAe,MAAM,oDASnD,CAAC;AAQF,eAAO,MAAM,6BAA6B,QAAe,MAAM,oBAAoB,MAAM,kBA+DxF,CAAC"}

package/dist/lib/rss/parser.js

@@ -66,6 +66,7 @@ const parseRSSFeedAndSaveToDatabase = (url, podcast_index_id) => __awaiter(void
         throw new Error(`parseRSSFeedAndSaveToDatabase: feed not found for ${url}`);
     }
     const parsedFeed = yield (0, feed_1.handleGetRSSFeed)(feed);
+    podverse_helpers_1.logger.info(`item count: ${parsedFeed.items.length}`);
     feed = yield (0, feed_1.handleParsedFeed)(parsedFeed, feed);
     try {
         yield feedService.update(feed.id, { is_parsing: new Date() });
@@ -89,13 +90,11 @@ const parseRSSFeedAndSaveToDatabase = (url, podcast_index_id) => __awaiter(void
         yield feedLogService.update(feed, { last_finished_parse_time: new Date() });
     }
     catch (error) {
-        const err = error;
-        podverse_helpers_1.logger.error('parseRSSFeedAndSaveToDatabase error:', err.stack);
-        throw err;
+        (0, podverse_helpers_1.logError)('parseRSSFeedAndSaveToDatabase', error);
     }
     finally {
         yield feedService.update(feed.id, { is_parsing: null });
     }
-    return feed;
+    return;
 });
 exports.parseRSSFeedAndSaveToDatabase = parseRSSFeedAndSaveToDatabase;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "podverse-parser",
-  "version": "5.0.7",
+  "version": "5.0.9",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",