webring 0.0.1

package/README.md

@@ -0,0 +1,24 @@
+# webring
+
+`webring` gathers the latest posts from your favorite RSS feeds so that you can embed them on your site.
+
+Inspired by:
+
+- https://github.com/lukehsiao/openring-rs
+- https://git.sr.ht/~sircmpwn/openring
+
+## Quick Start
+
+This library is meant to be used with static site generators. It is framework agnostic.
+
+### Astro
+
+### Command Line
+
+You can use this library to generate static HTML which can be included into your site using it's include mechanisms
+
+## Configuration
+
+```typescript
+
+```

package/dist/cache.d.ts

@@ -0,0 +1,3 @@
+import type { Configuration, Cache, Result, ResultEntry, Source } from "./types.js";
+export declare function runWithCache(config: Configuration, cache: Cache): Promise<[Result, Cache]>;
+export declare function fetchWithCache(source: Source, cache: Cache, config: Configuration): Promise<ResultEntry | undefined>;

package/dist/cache.js

@@ -0,0 +1,24 @@
+import * as R from "remeda";
+export async function runWithCache(config, cache) {
+    const promises = R.pipe(config.sources, R.map((source) => fetchWithCache(source, cache, config)), R.filter((result) => result !== undefined));
+    const results = await Promise.all(promises);
+    const definedResults = results.filter((result) => result !== undefined);
+    const updatedCache = R.pipe(definedResults, R.map((result) => [result.source.url, { timestamp: new Date(), data: result }]), R.fromEntries());
+    const topResults = R.pipe(definedResults, R.sortBy((result) => result.date.getTime()), R.reverse(), R.take(config.number));
+    return [topResults, updatedCache];
+}
+export async function fetchWithCache(source, cache, config) {
+    const cacheEntry = cache[source.url];
+    if (cacheEntry) {
+        const now = new Date();
+        if (now.getTime() - cacheEntry.timestamp.getTime() < config.cache_duration_minutes * 60 * 1000) {
+            console.log(`Cache entry found for ${source.url}`);
+            return Promise.resolve(cacheEntry.data);
+        }
+        else {
+            console.log(`Cache entry for ${source.url} is too old`);
+        }
+    }
+    console.log(`No cache entry for ${source.url}`);
+    return fetch(source, config.truncate);
+}

package/dist/fetch.d.ts

@@ -0,0 +1,2 @@
+import { type Source, type ResultEntry } from "./types.js";
+export declare function fetch(source: Source, length: number): Promise<ResultEntry | undefined>;

package/dist/fetch.js

@@ -0,0 +1,27 @@
+import Parser from "rss-parser";
+import sanitizeHtml from "sanitize-html";
+import truncate from "truncate-html";
+import { FeedEntrySchema } from "./types.js";
+import * as R from "remeda";
+export async function fetch(source, length) {
+    const parser = new Parser();
+    try {
+        const feed = await parser.parseURL(source.url);
+        const firstItem = R.pipe(feed.items, R.map((item) => FeedEntrySchema.parse(item)), R.sortBy((item) => new Date(item.date).getTime()), R.reverse(), R.first());
+        if (!firstItem) {
+            throw new Error("no items found in feed");
+        }
+        const preview = firstItem.contentSnippet ?? firstItem.content ?? firstItem.description ?? firstItem["content:encoded"];
+        return {
+            title: firstItem.title,
+            url: firstItem.link,
+            date: new Date(firstItem.date),
+            source,
+            preview: preview ? truncate(sanitizeHtml(preview), length) : undefined,
+        };
+    }
+    catch (e) {
+        console.error(`Error fetching ${source.url}: ${e}`);
+        return undefined;
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+import { type Configuration, type Result } from "./types.js";
+export declare function run(config: Configuration): Promise<Result>;

package/dist/index.js

@@ -0,0 +1,21 @@
+import { runWithCache } from "./cache.js";
+import { CacheSchema } from "./types.js";
+import fs from "fs/promises";
+export async function run(config) {
+    const cacheFilename = "cache.json";
+    const currentDir = process.cwd();
+    const fullFilename = `${currentDir}/${cacheFilename}`;
+    let cacheObject = {};
+    try {
+        const cacheFile = await fs.readFile(fullFilename);
+        cacheObject = CacheSchema.parse(JSON.parse(cacheFile.toString()));
+    }
+    catch (e) {
+        console.error("Error reading cache file:", e);
+        throw e;
+    }
+    const [result, updatedCache] = await runWithCache(config, cacheObject);
+    // write the updated cache to cache.json
+    await fs.writeFile(fullFilename, JSON.stringify(updatedCache));
+    return result;
+}

package/dist/types.d.ts

@@ -0,0 +1,284 @@
+import { z } from "zod";
+export type Source = z.infer<typeof SourceSchema>;
+declare const SourceSchema: z.ZodObject<{
+    url: z.ZodString;
+    title: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    title: string;
+}, {
+    url: string;
+    title: string;
+}>;
+export type Configuration = z.infer<typeof ConfigurationSchema>;
+declare const ConfigurationSchema: z.ZodObject<{
+    sources: z.ZodArray<z.ZodObject<{
+        url: z.ZodString;
+        title: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        url: string;
+        title: string;
+    }, {
+        url: string;
+        title: string;
+    }>, "many">;
+    number: z.ZodNumber;
+    cache_duration_minutes: z.ZodNumber;
+    truncate: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    number: number;
+    sources: {
+        url: string;
+        title: string;
+    }[];
+    cache_duration_minutes: number;
+    truncate: number;
+}, {
+    number: number;
+    sources: {
+        url: string;
+        title: string;
+    }[];
+    cache_duration_minutes: number;
+    truncate: number;
+}>;
+export type ResultEntry = z.infer<typeof ResultEntrySchema>;
+declare const ResultEntrySchema: z.ZodObject<{
+    title: z.ZodString;
+    url: z.ZodString;
+    date: z.ZodDate;
+    source: z.ZodObject<{
+        url: z.ZodString;
+        title: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        url: string;
+        title: string;
+    }, {
+        url: string;
+        title: string;
+    }>;
+    preview: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    title: string;
+    date: Date;
+    source: {
+        url: string;
+        title: string;
+    };
+    preview?: string | undefined;
+}, {
+    url: string;
+    title: string;
+    date: Date;
+    source: {
+        url: string;
+        title: string;
+    };
+    preview?: string | undefined;
+}>;
+export type Result = z.infer<typeof ResultSchema>;
+declare const ResultSchema: z.ZodArray<z.ZodObject<{
+    title: z.ZodString;
+    url: z.ZodString;
+    date: z.ZodDate;
+    source: z.ZodObject<{
+        url: z.ZodString;
+        title: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        url: string;
+        title: string;
+    }, {
+        url: string;
+        title: string;
+    }>;
+    preview: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    title: string;
+    date: Date;
+    source: {
+        url: string;
+        title: string;
+    };
+    preview?: string | undefined;
+}, {
+    url: string;
+    title: string;
+    date: Date;
+    source: {
+        url: string;
+        title: string;
+    };
+    preview?: string | undefined;
+}>, "many">;
+export type CacheEntry = z.infer<typeof CacheEntrySchema>;
+export declare const CacheEntrySchema: z.ZodObject<{
+    timestamp: z.ZodDate;
+    data: z.ZodObject<{
+        title: z.ZodString;
+        url: z.ZodString;
+        date: z.ZodDate;
+        source: z.ZodObject<{
+            url: z.ZodString;
+            title: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            url: string;
+            title: string;
+        }, {
+            url: string;
+            title: string;
+        }>;
+        preview: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    }, {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    timestamp: Date;
+    data: {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    };
+}, {
+    timestamp: Date;
+    data: {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    };
+}>;
+export type Cache = z.infer<typeof CacheSchema>;
+export declare const CacheSchema: z.ZodRecord<z.ZodString, z.ZodObject<{
+    timestamp: z.ZodDate;
+    data: z.ZodObject<{
+        title: z.ZodString;
+        url: z.ZodString;
+        date: z.ZodDate;
+        source: z.ZodObject<{
+            url: z.ZodString;
+            title: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            url: string;
+            title: string;
+        }, {
+            url: string;
+            title: string;
+        }>;
+        preview: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    }, {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    timestamp: Date;
+    data: {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    };
+}, {
+    timestamp: Date;
+    data: {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    };
+}>>;
+export declare const FeedEntrySchema: z.ZodEffects<z.ZodObject<{
+    title: z.ZodString;
+    link: z.ZodString;
+    isoDate: z.ZodOptional<z.ZodDate>;
+    pubDate: z.ZodOptional<z.ZodDate>;
+    content: z.ZodOptional<z.ZodString>;
+    contentSnippet: z.ZodOptional<z.ZodString>;
+    "content:encoded": z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    title: string;
+    link: string;
+    isoDate?: Date | undefined;
+    pubDate?: Date | undefined;
+    content?: string | undefined;
+    contentSnippet?: string | undefined;
+    "content:encoded"?: string | undefined;
+    description?: string | undefined;
+}, {
+    title: string;
+    link: string;
+    isoDate?: Date | undefined;
+    pubDate?: Date | undefined;
+    content?: string | undefined;
+    contentSnippet?: string | undefined;
+    "content:encoded"?: string | undefined;
+    description?: string | undefined;
+}>, {
+    title: string;
+    link: string;
+    date: Date;
+    content: string | undefined;
+    contentSnippet: string | undefined;
+    description: string | undefined;
+    "content:encoded": string | undefined;
+}, {
+    title: string;
+    link: string;
+    isoDate?: Date | undefined;
+    pubDate?: Date | undefined;
+    content?: string | undefined;
+    contentSnippet?: string | undefined;
+    "content:encoded"?: string | undefined;
+    description?: string | undefined;
+}>;
+export {};

package/dist/types.js

@@ -0,0 +1,50 @@
+import { z } from "zod";
+const SourceSchema = z.object({
+    url: z.string(),
+    title: z.string(),
+});
+const ConfigurationSchema = z.object({
+    sources: SourceSchema.array(),
+    number: z.number(),
+    cache_duration_minutes: z.number(),
+    truncate: z.number(),
+});
+const ResultEntrySchema = z.object({
+    title: z.string(),
+    url: z.string(),
+    date: z.coerce.date(),
+    source: SourceSchema,
+    preview: z.string().optional(),
+});
+const ResultSchema = z.array(ResultEntrySchema);
+export const CacheEntrySchema = z.object({
+    timestamp: z.coerce.date(),
+    data: ResultEntrySchema,
+});
+export const CacheSchema = z.record(CacheEntrySchema);
+export const FeedEntrySchema = z
+    .object({
+    title: z.string(),
+    link: z.string(),
+    isoDate: z.coerce.date().optional(),
+    pubDate: z.coerce.date().optional(),
+    content: z.string().optional(),
+    contentSnippet: z.string().optional(),
+    "content:encoded": z.string().optional(),
+    description: z.string().optional(),
+})
+    .transform((entry) => {
+    const date = entry.isoDate ?? entry.pubDate;
+    if (!date) {
+        throw new Error("no date found in feed entry");
+    }
+    return {
+        title: entry.title,
+        link: entry.link,
+        date,
+        content: entry.content,
+        contentSnippet: entry.contentSnippet,
+        description: entry.description,
+        "content:encoded": entry["content:encoded"],
+    };
+});

package/eslint.config.js

@@ -0,0 +1,17 @@
+import eslint from "@eslint/js";
+import tseslint from "typescript-eslint";
+
+export default tseslint.config(
+  eslint.configs.recommended,
+  ...tseslint.configs.strictTypeChecked,
+  ...tseslint.configs.stylisticTypeChecked,
+  {
+    languageOptions: {
+      parserOptions: {
+        project: true,
+        tsconfigRootDir: import.meta.dirname,
+      },
+    },
+    ignores: ["*.js"],
+  },
+);

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "webring",
+  "type": "module",
+  "version": "0.0.1",
+  "scripts": {
+    "lint": "eslint .",
+    "build": "tsc",
+    "test": ""
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "dependencies": {
+    "remeda": "^2.0.0",
+    "rss-parser": "^3.13.0",
+    "sanitize-html": "^2.13.0",
+    "truncate-html": "^1.1.1",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.3.0",
+    "@tsconfig/node20": "^20.1.4",
+    "@tsconfig/strictest": "^2.0.5",
+    "@types/eslint__js": "^8.42.3",
+    "@types/node": "^20.12.13",
+    "@types/sanitize-html": "^2.11.0",
+    "@typescript-eslint/eslint-plugin": "^7.11.0",
+    "@typescript-eslint/parser": "^7.11.0",
+    "eslint": "^8.57.0",
+    "husky": "^9.0.11",
+    "lint-staged": "^15.2.5",
+    "prettier": "^3.2.5",
+    "typescript": "^5.4.5",
+    "typescript-eslint": "^7.11.0"
+  },
+  "lint-staged": {
+    "*.{js,jsx,ts,tsx}": "eslint --cache --fix",
+    "*": "prettier --ignore-unknown --write"
+  }
+}

package/src/cache.ts

@@ -0,0 +1,48 @@
+import * as R from "remeda";
+import type { Configuration, Cache, Result, ResultEntry, CacheEntry, Source } from "./types.js";
+
+export async function runWithCache(config: Configuration, cache: Cache): Promise<[Result, Cache]> {
+  const promises = R.pipe(
+    config.sources,
+    R.map((source) => fetchWithCache(source, cache, config)),
+    R.filter((result) => result !== undefined),
+  );
+  const results = await Promise.all(promises);
+
+  const definedResults = results.filter((result) => result !== undefined) as ResultEntry[];
+
+  const updatedCache: Cache = R.pipe(
+    definedResults,
+    R.map((result): [string, CacheEntry] => [result.source.url, { timestamp: new Date(), data: result }]),
+    R.fromEntries(),
+  );
+
+  const topResults = R.pipe(
+    definedResults,
+    R.sortBy((result) => result.date.getTime()),
+    R.reverse(),
+    R.take(config.number),
+  );
+
+  return [topResults, updatedCache];
+}
+
+export async function fetchWithCache(
+  source: Source,
+  cache: Cache,
+  config: Configuration,
+): Promise<ResultEntry | undefined> {
+  const cacheEntry = cache[source.url];
+  if (cacheEntry) {
+    const now = new Date();
+    if (now.getTime() - cacheEntry.timestamp.getTime() < config.cache_duration_minutes * 60 * 1000) {
+      console.log(`Cache entry found for ${source.url}`);
+      return Promise.resolve(cacheEntry.data);
+    } else {
+      console.log(`Cache entry for ${source.url} is too old`);
+    }
+  }
+
+  console.log(`No cache entry for ${source.url}`);
+  return fetch(source, config.truncate);
+}

package/src/fetch.ts

@@ -0,0 +1,38 @@
+import Parser from "rss-parser";
+import sanitizeHtml from "sanitize-html";
+import truncate from "truncate-html";
+import { type Source, type ResultEntry, FeedEntrySchema } from "./types.js";
+import * as R from "remeda";
+
+export async function fetch(source: Source, length: number): Promise<ResultEntry | undefined> {
+  const parser = new Parser();
+  try {
+    const feed = await parser.parseURL(source.url);
+
+    const firstItem = R.pipe(
+      feed.items,
+      R.map((item) => FeedEntrySchema.parse(item)),
+      R.sortBy((item) => new Date(item.date).getTime()),
+      R.reverse(),
+      R.first(),
+    );
+
+    if (!firstItem) {
+      throw new Error("no items found in feed");
+    }
+
+    const preview =
+      firstItem.contentSnippet ?? firstItem.content ?? firstItem.description ?? firstItem["content:encoded"];
+
+    return {
+      title: firstItem.title,
+      url: firstItem.link,
+      date: new Date(firstItem.date),
+      source,
+      preview: preview ? truncate(sanitizeHtml(preview), length) : undefined,
+    };
+  } catch (e) {
+    console.error(`Error fetching ${source.url}: ${e as string}`);
+    return undefined;
+  }
+}

package/src/index.ts

@@ -0,0 +1,26 @@
+import { runWithCache } from "./cache.js";
+import { type Configuration, type Result, type Cache, CacheSchema } from "./types.js";
+import fs from "fs/promises";
+
+export async function run(config: Configuration): Promise<Result> {
+  const cacheFilename = "cache.json";
+  const currentDir = process.cwd();
+  const fullFilename = `${currentDir}/${cacheFilename}`;
+
+  let cacheObject: Cache = {};
+
+  try {
+    const cacheFile = await fs.readFile(fullFilename);
+    cacheObject = CacheSchema.parse(JSON.parse(cacheFile.toString()));
+  } catch (e) {
+    console.error("Error reading cache file:", e);
+    throw e;
+  }
+
+  const [result, updatedCache] = await runWithCache(config, cacheObject);
+
+  // write the updated cache to cache.json
+  await fs.writeFile(fullFilename, JSON.stringify(updatedCache));
+
+  return result;
+}

package/src/types.ts

@@ -0,0 +1,63 @@
+import { z } from "zod";
+
+export type Source = z.infer<typeof SourceSchema>;
+const SourceSchema = z.object({
+  url: z.string(),
+  title: z.string(),
+});
+
+export type Configuration = z.infer<typeof ConfigurationSchema>;
+const ConfigurationSchema = z.object({
+  sources: SourceSchema.array(),
+  number: z.number(),
+  cache_duration_minutes: z.number(),
+  truncate: z.number(),
+});
+
+export type ResultEntry = z.infer<typeof ResultEntrySchema>;
+const ResultEntrySchema = z.object({
+  title: z.string(),
+  url: z.string(),
+  date: z.coerce.date(),
+  source: SourceSchema,
+  preview: z.string().optional(),
+});
+
+export type Result = z.infer<typeof ResultSchema>;
+const ResultSchema = z.array(ResultEntrySchema);
+
+export type CacheEntry = z.infer<typeof CacheEntrySchema>;
+export const CacheEntrySchema = z.object({
+  timestamp: z.coerce.date(),
+  data: ResultEntrySchema,
+});
+
+export type Cache = z.infer<typeof CacheSchema>;
+export const CacheSchema = z.record(CacheEntrySchema);
+
+export const FeedEntrySchema = z
+  .object({
+    title: z.string(),
+    link: z.string(),
+    isoDate: z.coerce.date().optional(),
+    pubDate: z.coerce.date().optional(),
+    content: z.string().optional(),
+    contentSnippet: z.string().optional(),
+    "content:encoded": z.string().optional(),
+    description: z.string().optional(),
+  })
+  .transform((entry) => {
+    const date = entry.isoDate ?? entry.pubDate;
+    if (!date) {
+      throw new Error("no date found in feed entry");
+    }
+    return {
+      title: entry.title,
+      link: entry.link,
+      date,
+      content: entry.content,
+      contentSnippet: entry.contentSnippet,
+      description: entry.description,
+      "content:encoded": entry["content:encoded"],
+    };
+  });

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": ["@tsconfig/strictest/tsconfig.json", "@tsconfig/node20/tsconfig.json"],
+  "exclude": ["eslint.config.js", "dist"],
+  "compilerOptions": {
+    "outDir": "dist",
+    "declaration": true
+  }
+}