podverse-parser 5.0.6 → 5.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/lib/compat/partytime/item.js +1 -1
  2. package/dist/lib/rss/item/item.d.ts.map +1 -1
  3. package/dist/lib/rss/item/item.js +13 -1
  4. package/dist/lib/rss/liveItem/liveItem.js +1 -1
  5. package/dist/lib/rss/parser.d.ts.map +1 -1
  6. package/dist/lib/rss/parser.js +1 -0
  7. package/package.json +1 -1
package/dist/lib/compat/partytime/item.js CHANGED
@@ -7,7 +7,7 @@ const value_1 = require("./value");
7
7
  const compatItemDto = (parsedItem) => {
8
8
  var _a, _b;
9
9
  return ({
10
- guid: ((_a = parsedItem.guid) === null || _a === void 0 ? void 0 : _a.slice(0, podverse_helpers_1.DATABASE_CONSTANTS.varchar_guid)) || null,
10
+ guid: ((_a = parsedItem.guid) === null || _a === void 0 ? void 0 : _a.slice(0, podverse_helpers_1.DATABASE_CONSTANTS.varchar_url)) || null,
11
11
  guid_enclosure_url: parsedItem.enclosure.url.slice(0, podverse_helpers_1.DATABASE_CONSTANTS.varchar_url),
12
12
  pubdate: parsedItem.pubDate || null,
13
13
  title: ((_b = parsedItem.title) === null || _b === void 0 ? void 0 : _b.slice(0, podverse_helpers_1.DATABASE_CONSTANTS.varchar_normal)) || null
package/dist/lib/rss/item/item.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"item.d.ts","sourceRoot":"","sources":["../../../../src/lib/rss/item/item.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5C,OAAO,EAAiB,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAe,MAAM,cAAc,CAAC;AAmBtG,eAAO,MAAM,iBAAiB,gBAAuB,OAAO,EAAE,WAAW,OAAO,sBAAsB,kBAAkB,kBAuBvH,CAAC;AAEF,eAAO,MAAM,gBAAgB,eACf,OAAO,WACV,OAAO,sBACI,kBAAkB,8BACV,aAAa,kBAkC1C,CAAC"}
1
+ {"version":3,"file":"item.d.ts","sourceRoot":"","sources":["../../../../src/lib/rss/item/item.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5C,OAAO,EAAiB,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAe,MAAM,cAAc,CAAC;AA+BtG,eAAO,MAAM,iBAAiB,gBAAuB,OAAO,EAAE,WAAW,OAAO,sBAAsB,kBAAkB,kBAyBvH,CAAC;AAEF,eAAO,MAAM,gBAAgB,eACf,OAAO,WACV,OAAO,sBACI,kBAAkB,8BACV,aAAa,kBAkC1C,CAAC"}
package/dist/lib/rss/item/item.js CHANGED
@@ -29,12 +29,24 @@ const itemTranscript_1 = require("@parser/lib/rss/item/itemTranscript");
29
29
  const itemTxt_1 = require("@parser/lib/rss/item/itemTxt");
30
30
  const itemValue_1 = require("@parser/lib/rss/item/itemValue");
31
31
  const itemChat_1 = require("@parser/lib/rss/item/itemChat");
32
+ const removeDuplicates = (parsedItems) => {
33
+ const seen = new Set();
34
+ return parsedItems.reduce((acc, item) => {
35
+ const enclosureUrl = item.enclosure.url.slice(0, podverse_helpers_1.DATABASE_CONSTANTS.varchar_url);
36
+ if (!seen.has(enclosureUrl)) {
37
+ seen.add(enclosureUrl);
38
+ acc.push(item);
39
+ }
40
+ return acc;
41
+ }, []);
42
+ };
32
43
  const handleParsedItems = (parsedItems, channel, channelSeasonIndex) => __awaiter(void 0, void 0, void 0, function* () {
33
44
  const itemService = new podverse_orm_1.ItemService();
34
45
  const existingItems = yield itemService.getAllItemsByChannel(channel, { select: ['id'] });
35
46
  const existingItemIds = existingItems.map(item => item.id);
36
47
  const updatedItemIds = [];
37
- const parsedItemBatchs = (0, podverse_helpers_1.chunkArray)(parsedItems, 100);
48
+ const uniqueParsedItems = removeDuplicates(parsedItems);
49
+ const parsedItemBatchs = (0, podverse_helpers_1.chunkArray)(uniqueParsedItems, 50);
38
50
  for (const parsedItemBatch of parsedItemBatchs) {
39
51
  yield podverse_orm_1.AppDataSource.manager.transaction((transactionalEntityManager) => __awaiter(void 0, void 0, void 0, function* () {
40
52
  for (const parsedItem of parsedItemBatch) {
package/dist/lib/rss/liveItem/liveItem.js CHANGED
@@ -20,7 +20,7 @@ const handleParsedLiveItems = (parsedLiveItems, channel, channelSeasonIndex) =>
20
20
  const existingLiveItemIds = existingLiveItems.map(live_item => live_item.id);
21
21
  const updatedLiveItemIds = [];
22
22
  const liveItemObjDtos = (0, liveItem_1.compatLiveItemsDtos)(parsedLiveItems);
23
- const liveItemObjDtosBatchs = (0, podverse_helpers_1.chunkArray)(liveItemObjDtos, 100);
23
+ const liveItemObjDtosBatchs = (0, podverse_helpers_1.chunkArray)(liveItemObjDtos, 50);
24
24
  for (const liveItemObjDtosBatch of liveItemObjDtosBatchs) {
25
25
  yield podverse_orm_1.AppDataSource.manager.transaction((transactionalEntityManager) => __awaiter(void 0, void 0, void 0, function* () {
26
26
  for (const liveItemObjDto of liveItemObjDtosBatch) {
package/dist/lib/rss/parser.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../../src/lib/rss/parser.ts"],"names":[],"mappings":"AAeA,eAAO,MAAM,gBAAgB,oBAM5B,CAAC;AAEF,eAAO,MAAM,kBAAkB,QAAe,MAAM,oDASnD,CAAC;AAQF,eAAO,MAAM,6BAA6B,QAAe,MAAM,oBAAoB,MAAM,iBAgExF,CAAC"}
1
+ {"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../../src/lib/rss/parser.ts"],"names":[],"mappings":"AAeA,eAAO,MAAM,gBAAgB,oBAM5B,CAAC;AAEF,eAAO,MAAM,kBAAkB,QAAe,MAAM,oDASnD,CAAC;AAQF,eAAO,MAAM,6BAA6B,QAAe,MAAM,oBAAoB,MAAM,iBAiExF,CAAC"}
package/dist/lib/rss/parser.js CHANGED
@@ -66,6 +66,7 @@ const parseRSSFeedAndSaveToDatabase = (url, podcast_index_id) => __awaiter(void
66
66
  throw new Error(`parseRSSFeedAndSaveToDatabase: feed not found for ${url}`);
67
67
  }
68
68
  const parsedFeed = yield (0, feed_1.handleGetRSSFeed)(feed);
69
+ podverse_helpers_1.logger.info(`item count: ${parsedFeed.items.length}`);
69
70
  feed = yield (0, feed_1.handleParsedFeed)(parsedFeed, feed);
70
71
  try {
71
72
  yield feedService.update(feed.id, { is_parsing: new Date() });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "podverse-parser",
3
- "version": "5.0.6",
3
+ "version": "5.0.8",
4
4
  "description": "",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",