podverse-parser 1.0.0 → 5.0.0-beta.1

package/eslint.config.mjs

@@ -0,0 +1,38 @@
+import globals from "globals";
+import pluginJs from "@eslint/js";
+import tseslint from "@typescript-eslint/eslint-plugin";
+import tsParser from "@typescript-eslint/parser";
+
+const pluginJsRecommended = {
+  rules: {
+    ...pluginJs.configs.recommended.rules,
+  },
+};
+
+const tseslintRecommended = {
+  rules: {
+    ...tseslint.configs.recommended.rules,
+  },
+};
+
+export default [
+  {
+    files: ["src/**/*.{js,ts}"],
+    languageOptions: {
+      globals: globals.node,
+      parser: tsParser,
+    },
+    plugins: {
+      "@typescript-eslint": tseslint,
+    },
+    rules: {
+      ...pluginJsRecommended.rules,
+      ...tseslintRecommended.rules,
+      "semi": ["error", "always"],
+      "indent": ["error", 2],
+      "@typescript-eslint/no-require-imports": "off",
+    },
+    ignores: ["**/eslint.config.mjs"],
+  }
+];
+

package/package.json

@@ -1,41 +1,32 @@
 {
   "name": "podverse-parser",
-  "version": "1.0.0",
-  "description": "The RSS parser system for podverse-parser",
+  "version": "5.0.0-beta.1",
+  "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
-    "dev": "nodemon --watch ./src -x \"npm run build\"",
-    "build": "esbuild src/index.ts --bundle --minify --sourcemap --platform=node --target=es2020 --outfile=dist/index.js",
+    "dev:watch": "nodemon --watch 'src' --watch $(realpath node_modules/podverse-external-services) --watch $(realpath node_modules/podverse-orm) --watch $(realpath node_modules/podverse-shared) -x \"npm run build\"",
+    "build": "tsc",
     "lint": "eslint ./src --ext .ts",
-    "lint-fix": "eslint --fix",
-    "prepare": "npm run build"
-  },
-  "files": [
-    "/dist"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/podverse/podverse-parser.git"
+    "prepare": "npm run build",
+    "start": "ts-node ./dist/index.js"
   },
+  "author": "",
   "license": "AGPLv3",
-  "bugs": {
-    "url": "https://github.com/podverse/podverse-parser/issues"
+  "dependencies": {
+    "module-alias": "^2.2.3",
+    "podcast-partytime": "^4.8.1",
+    "podverse-helpers": "^5.0.0-beta.1",
+    "podverse-orm": "^5.0.0-beta.1"
   },
-  "homepage": "https://github.com/podverse/podverse-parser#readme",
   "devDependencies": {
-    "@typescript-eslint/eslint-plugin": "^6.13.0",
-    "@typescript-eslint/parser": "^6.13.0",
-    "esbuild": "^0.19.8",
-    "eslint": "^8.54.0",
-    "nodemon": "^3.0.2",
-    "typescript": "^5.3.2"
-  },
-  "dependencies": {
-    "aws-sdk": "2.814.0",
-    "axios": "^1.6.2",
-    "podverse-shared": "^4.15.2",
-    "request-promise-native": "1.0.8",
-    "sharp": "^0.33.0"
+    "@eslint/config-array": "^0.18.0",
+    "@eslint/eslintrc": "^3.1.0",
+    "@eslint/js": "^9.8.0",
+    "@types/node": "^22.1.0",
+    "eslint": "^9.9.0",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.5.4",
+    "typescript-eslint": "^8.0.1"
   }
 }

package/src/index.ts

@@ -0,0 +1,3 @@
+import './module-alias-config';
+
+export { parseAllRSSFeeds, parseRSSFeedAndSaveToDatabase } from './lib/rss/parser';

package/src/lib/chapters/chapters.ts

@@ -0,0 +1,54 @@
+import { request, throwRequestError } from "podverse-helpers";
+import { Item, ItemChaptersFeed, ItemChapterService, ItemChaptersFeedLogService } from "podverse-orm";
+import { compatParsedChapters, PIChapter } from "@parser/lib/compat/chapters/chapters";
+
+const getParsedChapters = async (item_chapters_feed: ItemChaptersFeed) => {
+  const itemChaptersFeedLogService = new ItemChaptersFeedLogService();
+  try {
+    const response: { chapters: PIChapter[] } = await request(item_chapters_feed.url);
+    await itemChaptersFeedLogService.update(item_chapters_feed, {
+      last_http_status: 200,
+      last_good_http_status_time: new Date()
+    });
+    return compatParsedChapters(response.chapters);
+  } catch (error) {
+    // TODO: how to handle errors?
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const statusCode = (error as any).statusCode as number;
+    const item_chapters_feed_log = await itemChaptersFeedLogService.get(item_chapters_feed);
+    if (statusCode) {
+      await itemChaptersFeedLogService.update(item_chapters_feed, {
+        last_http_status: statusCode,
+        parse_errors: (item_chapters_feed_log?.parse_errors || 0) + 1,
+      });
+    }
+    return throwRequestError(error);
+  }
+};
+
+export const parseChapters = async (item: Item): Promise<void> => {
+  const item_chapters_feed = item.item_chapters_feed;
+
+  if (!item_chapters_feed) {
+    return;
+  }
+
+  const parsedChapters = await getParsedChapters(item_chapters_feed);
+  
+  const itemChapterService = new ItemChapterService();
+
+  const existingChapters = await itemChapterService.getAll(item_chapters_feed, { select: ['id'] });
+  const existingChaptersIds = existingChapters.map(item_chapter => item_chapter.id);
+  const updatedChaptersIds: number[] = [];
+
+  for (const parsedChapter of parsedChapters) {
+    await itemChapterService.update(item_chapters_feed, parsedChapter);
+    updatedChaptersIds.push(item_chapters_feed.id);
+  }
+
+  const itemChapterIdsToDelete = existingChaptersIds.filter(id => !updatedChaptersIds.includes(id));
+  await itemChapterService.deleteMany(itemChapterIdsToDelete);
+
+  const itemChaptersFeedLogService = new ItemChaptersFeedLogService();
+  await itemChaptersFeedLogService.update(item_chapters_feed, { last_finished_parse_time: new Date() });
+};

package/src/lib/compat/channel.ts

@@ -0,0 +1,287 @@
+import { FeedObject, Phase4Medium } from "podcast-partytime";
+import { Phase4PodcastImage } from "podcast-partytime/dist/parser/phase/phase-4";
+import { createSortableTitle, getBooleanOrNull } from "podverse-helpers";
+import { getChannelItunesTypeItunesTypeEnumValue, getMediumEnumValue } from "podverse-orm";
+import { compatChannelValue } from "@parser/lib/compat/partytime/value";
+
+export const compatChannelDto = (parsedFeed: FeedObject) => ({
+  podcast_guid: parsedFeed.guid,
+  title: parsedFeed.title,
+  sortable_title: createSortableTitle(parsedFeed.title),
+  medium: getMediumEnumValue(parsedFeed.medium ?? Phase4Medium.Podcast)
+});
+
+export const compatChannelAboutDto = (parsedFeed: FeedObject) => ({
+  author: (Array.isArray(parsedFeed.author) ? parsedFeed.author : parsedFeed.author ? [parsedFeed.author] : [])?.join(', ') || null,
+  explicit: getBooleanOrNull(parsedFeed.explicit),
+  language: parsedFeed.language || null,
+  website_link_url: parsedFeed.link || null,
+  itunes_type: getChannelItunesTypeItunesTypeEnumValue(parsedFeed.itunesType || 'episodic')
+});
+
+export const compatChannelChatDto = (parsedFeed: FeedObject) => {
+  if (!parsedFeed.chat) {
+    return null;
+  }
+  return {
+    server: parsedFeed.chat.server,
+    protocol: parsedFeed.chat.protocol,
+    account_id: parsedFeed.chat.accountId || null,
+    space: parsedFeed.chat.space || null
+  };
+};
+
+export const compatChannelDescriptionDto = (parsedFeed: FeedObject) => {
+  if (!parsedFeed.description) {
+    return null;
+  }
+  return {
+    value: parsedFeed.description
+  };
+};
+
+export const compatChannelFundingDtos = (parsedFeed: FeedObject) => {
+  const dtos = [];
+
+  if (Array.isArray(parsedFeed.podcastFunding)) {
+    for (const f of parsedFeed.podcastFunding) {
+      if (f.url) {
+        dtos.push({
+          url: f.url,
+          title: f.message || null
+        });
+      }
+    }
+  }
+
+  return dtos;
+};
+
+export const compatChannelImageDtos = (parsedFeed: FeedObject) => {
+  const dtos = [];
+  if (parsedFeed.itunesImage) {
+    dtos.push({
+      url: parsedFeed.itunesImage,
+      image_width_size: null
+    });
+  } else if (parsedFeed.image?.url) {
+    dtos.push({
+      url: parsedFeed.image.url,
+      image_width_size: null
+    });
+  }
+
+  function hasWidth(image: Phase4PodcastImage['parsed']): image is { url: string; width: number } {
+    return (image as { width: number }).width !== undefined;
+  }
+
+  if (Array.isArray(parsedFeed.podcastImages)) {
+    for (const image of parsedFeed.podcastImages) {
+      if (image.parsed.url && hasWidth(image.parsed)) {
+        dtos.push({
+          url: image.parsed.url,
+          image_width_size: image.parsed.width
+        });
+      }
+    }
+  }
+
+  return dtos;
+};
+
+export const compatChannelLicenseDto = (parsedFeed: FeedObject) => {
+  if (!parsedFeed?.license?.identifier) {
+    return null;
+  }
+  return {
+    identifier: parsedFeed.license.identifier,
+    url: parsedFeed.license.url || null
+  };
+};
+
+export const compatChannelLocationDto = (parsedFeed: FeedObject) => {
+  if (!parsedFeed?.podcastLocation?.geo && !parsedFeed?.podcastLocation?.osm) {
+    return null;
+  }
+
+  return {
+    geo: parsedFeed.podcastLocation.geo || null,
+    osm: parsedFeed.podcastLocation.osm || null,
+    name: parsedFeed.podcastLocation.name || null
+  };
+};
+
+export const compatChannelPersonDtos = (parsedFeed: FeedObject) => {
+  const dtos = [];
+
+  if (Array.isArray(parsedFeed.podcastPeople)) {
+    for (const p of parsedFeed.podcastPeople) {
+      if (p.name) {
+        dtos.push({
+          name: p.name,
+          role: p.role?.toLowerCase() || null,
+          person_group: p.group?.toLowerCase() || 'cast',
+          img: p.img || null,
+          href: p.href || null
+        });
+      }
+    }
+  }
+
+  return dtos;
+};
+
+export const compatChannelPodrollRemoteItemDtos = (parsedFeed: FeedObject) => {
+  const dtos = [];
+
+  if (Array.isArray(parsedFeed.podroll)) {
+    for (const ri of parsedFeed.podroll) {
+      if (ri.feedGuid) {
+        dtos.push({
+          feed_guid: ri.feedGuid,
+          feed_url: ri.feedUrl || null,
+          item_guid: null,
+          title: /* PTDO: ri.title || */ null
+        });
+      }
+    }
+  }
+
+  return dtos;
+};
+
+export const compatChannelPublisherRemoteItemDtos = (parsedFeed: FeedObject) => {
+  const dtos = [];
+
+  if (Array.isArray(parsedFeed.podroll)) {
+    for (const ri of parsedFeed.podroll) {
+      if (ri.feedGuid) {
+        dtos.push({
+          feed_guid: ri.feedGuid,
+          feed_url: ri.feedUrl || null,
+          item_guid: null,
+          title: /* PTDO: ri.title || */ null
+        });
+      }
+    }
+  }
+
+  return dtos;
+};
+
+export const compatChannelRemoteItemDtos = (parsedFeed: FeedObject) => {
+  const dtos = [];
+
+  if (Array.isArray(parsedFeed.podcastRemoteItems)) {
+    for (const ri of parsedFeed.podcastRemoteItems) {
+      if (ri.feedGuid) {
+        dtos.push({
+          feed_guid: ri.feedGuid,
+          feed_url: ri.feedUrl || null,
+          item_guid: null,
+          title: /* PTDO: ri.title || */ null
+        });
+      }
+    }
+  }
+
+  return dtos;
+};
+
+export const compatChannelSocialInteractDtos = (parsedFeed: FeedObject) => {
+  const dtos = [];
+
+  if (parsedFeed?.podcastSocial?.length) {
+    for (const ps of parsedFeed.podcastSocial) {
+      dtos.push({
+        // PTDO: fix keys mismatch between partytime and podverse
+        protocol: ps.platform,
+        uri: ps.url,
+        account_id: ps.id || null,
+        account_url: ps.name || null,
+        priority: ps.priority || null
+      });
+    }
+  }
+
+  return dtos;
+};
+
+export const compatChannelSeasonDtos = (parsedFeed: FeedObject) => {
+  const dtos = [];
+
+  const parsedItems = parsedFeed?.items || [];
+
+  const seasonsIndex: { [key: number]: { name: string | null } } = {};
+
+  for (const parsedItem of parsedItems) {
+    const seasonNumber = parsedItem?.podcastSeason?.number || parsedItem?.itunesSeason;
+    const seasonName = parsedItem?.podcastSeason?.name || null;
+    if (Number.isInteger(seasonNumber)) {
+      const seasonNumberAsNumber = seasonNumber as number;
+      seasonsIndex[seasonNumberAsNumber] = {
+        name: seasonName
+      };
+    }
+  }
+
+  for (const [number, { name }] of Object.entries(seasonsIndex)) {
+    dtos.push({
+      number: parseInt(number),
+      name: name || null
+    });
+  }
+
+  return dtos;
+};
+
+export const compatChannelTrailerDtos = (parsedFeed: FeedObject) => {
+  const dtos = [];
+  if (parsedFeed?.trailers?.length) {
+    for (const pt of parsedFeed.trailers) {
+      dtos.push({
+        url: pt.url,
+        title: /* PTDO: add pt.title || */ null,
+        pubdate: pt.pubdate,
+        length: pt.length || null,
+        type: pt.type || null,
+        season: pt.season || null
+      });
+    }
+  }
+
+  return dtos;
+};
+
+export const compatChannelTxtDtos = (parsedFeed: FeedObject) => {
+  const dtos = [];
+  if (parsedFeed?.podcastTxt?.length) {
+    for (const pt of parsedFeed.podcastTxt) {
+      dtos.push({
+        purpose: pt.purpose || null,
+        value: pt.value
+      });
+    }
+  }
+
+  return dtos;
+};
+
+export const compatChannelValueDtos = (parsedFeed: FeedObject) => {
+  let dtos = [];
+  if (parsedFeed.value) {
+    const dto = compatChannelValue(parsedFeed.value);
+
+    const formattedDto = {
+      channel_value: {
+        type: dto.type,
+        method: dto.method,
+        suggested: dto.suggested || null
+      },
+      channel_value_recipients: dto.channel_value_recipients
+    };
+
+    dtos.push(formattedDto);
+  }
+  return dtos;
+};

package/src/lib/compat/chapters/chapters.ts

@@ -0,0 +1,23 @@
+import { ItemChapterDto } from 'podverse-orm';
+
+export type PIChapter = {
+  startTime: string
+  endTime: string | null
+  title: string | null
+  img: string | null
+  url: string | null
+  toc: boolean
+}
+
+export const compatParsedChapters = (chapters: PIChapter[]): ItemChapterDto[] => {
+  return chapters.map(chapter => {
+    return {
+      start_time: chapter.startTime,
+      end_time: chapter.endTime || null,
+      title: chapter.title || null,
+      img: chapter.img || null,
+      web_url: chapter.url || null,
+      table_of_contents: chapter.toc || true
+    };
+  });
+};