webring 0.0.0-dev.706

package/src/__snapshots__/index.test.ts.snap

@@ -0,0 +1,7 @@
+// Bun Snapshot v1, https://bun.sh/docs/test/snapshots
+
+exports[`it should fetch an RSS feed without caching 1`] = `"[{"title":"Creating an already-completed asynchronous activity in C++/WinRT, part 4","url":"https://devblogs.microsoft.com/oldnewthing/20240712-00/?p=109967","date":"2024-07-12T14:00:00.000Z","source":{"title":"rss 10","url":"http://localhost:PORT/rss-10.xml"},"preview":"Failing is easy. Failing correctly is hard. The post Creating an already-completed asynchronous activity in C++/WinRT, part 4 appeared first on The Old New Thing."}]"`;
+
+exports[`it should fetch several RSS feeds 1`] = `"[{"title":"Creating an already-completed asynchronous activity in C++/WinRT, part 4","url":"https://devblogs.microsoft.com/oldnewthing/20240712-00/?p=109967","date":"2024-07-12T14:00:00.000Z","source":{"title":"rss 10","url":"http://localhost:PORT/rss-10.xml"},"preview":"Failing is easy. Failing correctly is hard. The post Creating an already-completed asynchronous activity in C++/WinRT, part 4 appeared first on The Old New Thing."},{"title":"Garlic and gravel","url":"https://www.henrikkarlsson.xyz/p/garlic-and-gravel","date":"2024-07-12T11:34:25.000Z","source":{"title":"rss 16","url":"http://localhost:PORT/rss-16.xml"},"preview":"fragments"},{"title":"My Glorious Ascension To Thought Leadership","url":"https://ludic.mataroa.blog/blog/my-glorious-ascension-to-thought-leadership/","date":"2024-07-11T00:00:00.000Z","source":{"title":"rss 5","url":"http://localhost:PORT/rss-5.xml"},"preview":"I. This Escalated Quickly I mentioned one post ago that this blog has done approximately 2M+ lifetime views, with over a million of them being in the past two weeks in response to the now-infamous rant about AI, then I went out of my way to not write the most maximally viral thing possible for perso..."}]"`;
+
+exports[`it should fetch an RSS feed with caching 1`] = `"[{"title":"Creating an already-completed asynchronous activity in C++/WinRT, part 4","url":"https://devblogs.microsoft.com/oldnewthing/20240712-00/?p=109967","date":"2024-07-12T14:00:00.000Z","source":{"url":"http://localhost:PORT/rss-10.xml","title":"rss 10"},"preview":"Failing is easy. Failing correctly is hard. The post Creating an already-completed asynchronous activity in C++/WinRT, part 4 appeared first on The Old New Thing."}]"`;

package/src/cache.ts

@@ -0,0 +1,99 @@
+import * as R from "remeda";
+import {
+  type Cache,
+  type Result,
+  type ResultEntry,
+  type CacheEntry,
+  type Source,
+  type CachedConfiguration,
+  CacheSchema,
+  type CacheConfiguration,
+} from "./types.ts";
+import { fetch } from "./fetch.ts";
+import { asyncMapFilterUndefined } from "./util.ts";
+import * as fs from "node:fs/promises";
+
+async function loadCache({
+  cache_file: cacheFilePath,
+}: CacheConfiguration): Promise<Cache> {
+  try {
+    await fs.access(cacheFilePath);
+    const cacheFileContent = await fs.readFile(cacheFilePath);
+    return CacheSchema.parse(JSON.parse(cacheFileContent.toString()));
+  } catch {
+    return {};
+  }
+}
+
+async function saveCache(
+  { cache_file: cacheFilePath }: CacheConfiguration,
+  cache: Cache,
+) {
+  const dir = cacheFilePath.split("/").slice(0, -1).join("/");
+  if (dir !== "") {
+    await fs.mkdir(cacheFilePath.split("/").slice(0, -1).join("/"), {
+      recursive: true,
+    });
+  }
+  await fs.writeFile(cacheFilePath, JSON.stringify(cache));
+}
+
+function toCacheEntry(result: ResultEntry, now: Date): [string, CacheEntry] {
+  return [result.source.url, { timestamp: now, data: result }];
+}
+
+function toCache(results: ResultEntry[], now: Date): Cache {
+  return R.pipe(
+    results,
+    R.map((result) => toCacheEntry(result, now)),
+    R.fromEntries(),
+  );
+}
+
+function updateCache(
+  results: ResultEntry[],
+  config: CachedConfiguration,
+): Promise<void> {
+  const now = new Date();
+  const updatedCache = toCache(results, now);
+  return saveCache(config.cache, updatedCache);
+}
+
+export async function fetchAllCached(
+  config: CachedConfiguration,
+): Promise<Result> {
+  const cache = await loadCache(config.cache);
+
+  const results = await asyncMapFilterUndefined(config.sources, (source) =>
+    fetchWithCache(source, cache, config),
+  );
+
+  await updateCache(results, config);
+
+  return results;
+}
+
+export async function fetchWithCache(
+  source: Source,
+  cache: Cache,
+  config: CachedConfiguration,
+): Promise<ResultEntry | undefined> {
+  const cacheEntry = cache[source.url];
+  if (cacheEntry) {
+    const now = new Date();
+    if (
+      now.getTime() - cacheEntry.timestamp.getTime() <
+      config.cache.cache_duration_minutes * 60 * 1000
+    ) {
+      console.warn(`Cache entry found for ${source.url}.`);
+      return cacheEntry.data;
+    } else {
+      console.warn(`Cache entry for ${source.url} is too old.`);
+    }
+  } else {
+    console.warn(`No cache entry for ${source.url}.`);
+  }
+
+  console.warn(`Fetching ${source.url}`);
+  return fetch(source, config.truncate);
+}

package/src/fetch.ts

@@ -0,0 +1,84 @@
+import Parser from "rss-parser";
+import sanitizeHtml from "sanitize-html";
+import * as truncateHtml from "truncate-html";
+import { z } from "zod/v3";
+import {
+  type Source,
+  type ResultEntry,
+  FeedEntrySchema,
+  type Configuration,
+} from "./types.ts";
+import * as R from "remeda";
+import { asyncMapFilterUndefined } from "./util.ts";
+
+// Handle ESM/CommonJS interop - truncate-html exports differently in different environments
+const TruncateFnSchema = z
+  .function()
+  .args(z.string(), z.number().optional())
+  .returns(z.string());
+const TruncateModuleSchema = z.object({ default: TruncateFnSchema });
+
+function truncate(html: string, length?: number): string {
+  const mod = truncateHtml as unknown;
+  const fnResult = TruncateFnSchema.safeParse(mod);
+  if (fnResult.success) {
+    return fnResult.data(html, length);
+  }
+  const modResult = TruncateModuleSchema.safeParse(mod);
+  if (modResult.success) {
+    return modResult.data.default(html, length);
+  }
+  throw new Error("truncate-html module could not be resolved");
+}
+
+export async function fetchAll(config: Configuration) {
+  return await asyncMapFilterUndefined(config.sources, (source) =>
+    fetch(source, config.truncate),
+  );
+}
+
+export async function fetch(
+  source: Source,
+  length: number,
+): Promise<ResultEntry | undefined> {
+  const parser = new Parser();
+
+  try {
+    const feed = await parser.parseURL(source.url);
+
+    const firstItem = R.pipe(
+      feed.items,
+      R.map((item) => FeedEntrySchema.parse(item)),
+      R.sortBy((item) => new Date(item.date).getTime()),
+      R.reverse(),
+      R.first(),
+    );
+
+    if (!firstItem) {
+      throw new Error("no items found in feed");
+    }
+
+    const preview =
+      firstItem.contentSnippet ??
+      firstItem.content ??
+      firstItem.description ??
+      firstItem["content:encoded"];
+
+    return {
+      title: firstItem.title,
+      url: firstItem.link,
+      date: new Date(firstItem.date),
+      source,
+      preview:
+        preview !== undefined && preview !== ""
+          ? truncate(
+              sanitizeHtml(preview, { parseStyleAttributes: false }),
+              length,
+            )
+          : undefined,
+    };
+  } catch (error) {
+    console.error(`Error fetching ${source.url}: ${String(error)}`);
+    return undefined;
+  }
+}

package/src/index.test.ts

@@ -0,0 +1,81 @@
+import { expect, test } from "bun:test";
+import type { Configuration } from "./types.ts";
+import { run } from "./index.ts";
+import { tmpdir } from "node:os";
+import { mkdtemp } from "node:fs/promises";
+import path from "node:path";
+import express from "express";
+
+const app = express();
+app.use(express.static("src/testdata"));
+
+const port = Math.floor(Math.random() * 10_000) + 3000;
+app.listen(port, () => {
+  console.warn(`Test server listening at http://localhost:${port.toString()}`);
+});
+
+await new Promise((resolve) => setTimeout(resolve, 500));
+
+function createUrl(urlPath: string): string {
+  return `http://localhost:${port.toString()}/${urlPath}`;
+}
+
+function createSources(count: number): Configuration["sources"] {
+  return Array.from({ length: count }, (_, i) => ({
+    title: `rss ${i.toString()}`,
+    url: createUrl(`rss-${i.toString()}.xml`),
+  }));
+}
+
+test("it should fetch an RSS feed without caching", async () => {
+  const config: Configuration = {
+    sources: createSources(19),
+    number: 1,
+    truncate: 300,
+  };
+
+  const result = await run(config);
+  let string = JSON.stringify(result);
+  // replace the port number with a fixed value
+  string = string.replaceAll(port.toString(), "PORT");
+  expect(string).toMatchSnapshot();
+});
+
+test("it should fetch several RSS feeds", async () => {
+  const config: Configuration = {
+    sources: createSources(19),
+    number: 3,
+    truncate: 300,
+  };
+
+  const result = await run(config);
+  let string = JSON.stringify(result);
+  // replace the port number with a fixed value
+  string = string.replaceAll(port.toString(), "PORT");
+  expect(string).toMatchSnapshot();
+});
+
+test("it should fetch an RSS feed with caching", async () => {
+  const config: Configuration = {
+    sources: createSources(19),
+    number: 1,
+    truncate: 300,
+    cache: {
+      cache_file: `${await createTempDir()}/cache.json`,
+      cache_duration_minutes: 1,
+    },
+  };
+
+  const result = await run(config);
+  let string = JSON.stringify(result);
+  // replace the port number with a fixed value
+  string = string.replaceAll(port.toString(), "PORT");
+  expect(string).toMatchSnapshot();
+});
+
+// https://sdorra.dev/posts/2024-02-12-vitest-tmpdir
+async function createTempDir() {
+  const ostmpdir = tmpdir();
+  const dir = path.join(ostmpdir, "unit-test-");
+  return await mkdtemp(dir);
+}

package/src/index.ts

@@ -0,0 +1,50 @@
+import * as R from "remeda";
+import { fetchAllCached } from "./cache.ts";
+import { fetchAll as fetchAllUncached } from "./fetch.ts";
+import {
+  type Configuration,
+  type Result,
+  CachedConfigurationSchema,
+} from "./types.ts";
+
+export type { Configuration, Result, ResultEntry } from "./types.ts";
+
+export async function run(config: Configuration): Promise<Result> {
+  const { success, data } = CachedConfigurationSchema.safeParse(config);
+
+  let fetched: Result;
+  if (success) {
+    console.warn(`Using cache at ${data.cache.cache_file}.`);
+    fetched = await fetchAllCached(data);
+  } else {
+    console.warn("Cache disabled.");
+    fetched = await fetchAllUncached(config);
+  }
+
+  let results = R.pipe(
+    fetched,
+    R.sortBy((entry) => entry.date.getTime()),
+    R.reverse(),
+    R.filter((entry) => {
+      const filterFn = entry.source.filter;
+      if (
+        filterFn === undefined ||
+        entry.preview === undefined ||
+        entry.preview === ""
+      ) {
+        return true;
+      }
+      return filterFn(entry.preview);
+    }),
+  );
+
+  // shuffle if wanted
+  if (config.shuffle === true) {
+    results = R.shuffle(results);
+  }
+
+  // take n
+  results = R.take(results, config.number);
+
+  return results;
+}

package/src/types.ts

@@ -0,0 +1,103 @@
+import { z } from "zod/v3";
+
+export type Source = z.infer<typeof SourceSchema>;
+/** An RSS source */
+const SourceSchema = z.object({
+  /** The URL of an RSS feed */
+  url: z.string(),
+  /** A title to describe the feed */
+  title: z.string().describe("A title for the feed"),
+  /** Takes a entry preview and returns whether or not it should be displayed */
+  filter: z.function().args(z.string()).returns(z.boolean()).optional(),
+});
+
+export type CacheConfiguration = z.infer<typeof CacheConfigurationSchema>;
+/** Configuration for the cache */
+const CacheConfigurationSchema = z.object({
+  /** How long to cache a result for */
+  cache_duration_minutes: z.number().default(60),
+  /** The location of a file to use as a cache */
+  cache_file: z.string().default("cache.json"),
+});
+
+export type Configuration = z.infer<typeof ConfigurationSchema>;
+/** A configuration object with caching possibly configured */
+const ConfigurationSchema = z.object({
+  /** A list of sources to fetch */
+  sources: SourceSchema.array(),
+  /** Return the n latest updates from the source list. */
+  number: z.number().default(3),
+  /** How many words the preview field should be truncated to in characters after HTML has been sanitized and parsed. */
+  truncate: z.number().default(300),
+  /** Configuration for the cache */
+  cache: CacheConfigurationSchema.optional(),
+  /** Randomize the output order */
+  shuffle: z.boolean().default(false).optional(),
+});
+
+export type CachedConfiguration = z.infer<typeof CachedConfigurationSchema>;
+/** A configuration object with caching definitely configured */
+export const CachedConfigurationSchema = ConfigurationSchema.extend({
+  /** Configuration for the cache */
+  cache: CacheConfigurationSchema,
+});
+
+export type ResultEntry = z.infer<typeof ResultEntrySchema>;
+/** A single entry from an RSS feed */
+const ResultEntrySchema = z.object({
+  /** The title of the entry */
+  title: z.string(),
+  /** A direct link to the entry */
+  url: z.string(),
+  /** The date of the entry */
+  date: z.coerce.date(),
+  /** The source the entry is from */
+  source: SourceSchema,
+  /** A preview of the entry. This may contain sanitized HTML. */
+  preview: z.string().optional(),
+});
+
+export type Result = z.infer<typeof ResultSchema>;
+/** A list of results */
+export const ResultSchema = z.array(ResultEntrySchema);
+
+export type CacheEntry = z.infer<typeof CacheEntrySchema>;
+/** A single cache entry */
+export const CacheEntrySchema = z.object({
+  /** The time a source was last checked */
+  timestamp: z.coerce.date(),
+  /** The data from the source */
+  data: ResultEntrySchema,
+});
+
+export type Cache = z.infer<typeof CacheSchema>;
+/** A mapping of source URLs to cache entries */
+export const CacheSchema = z.record(CacheEntrySchema);
+
+/** The expected format fetched RSS feed entries */
+export const FeedEntrySchema = z
+  .object({
+    title: z.string(),
+    link: z.string(),
+    isoDate: z.coerce.date().optional(),
+    pubDate: z.coerce.date().optional(),
+    content: z.string().optional(),
+    contentSnippet: z.string().optional(),
+    "content:encoded": z.string().optional(),
+    description: z.string().optional(),
+  })
+  .transform((entry) => {
+    const date = entry.isoDate ?? entry.pubDate;
+    if (!date) {
+      throw new Error("no date found in feed entry");
+    }
+    return {
+      title: entry.title,
+      link: entry.link,
+      date,
+      content: entry.content,
+      contentSnippet: entry.contentSnippet,
+      description: entry.description,
+      "content:encoded": entry["content:encoded"],
+    };
+  });

package/src/util.ts

@@ -0,0 +1,32 @@
+import * as R from "remeda";
+
+// run an async map operation, filtering out undefined results
+export async function asyncMapFilterUndefined<T, U>(
+  input: T[],
+  fn: (x: T) => Promise<U | undefined>,
+): Promise<U[]> {
+  const results = await asyncMap(input, fn);
+  return filterUndefined(results);
+}
+
+export async function asyncMap<T, U>(
+  input: T[],
+  fn: (x: T) => Promise<U>,
+): Promise<U[]> {
+  const promises = R.pipe(
+    input,
+    R.map((item) => fn(item)),
+  );
+
+  return Promise.all(promises);
+}
+
+export function filterUndefined<T>(input: (T | undefined)[]): T[] {
+  const result: T[] = [];
+  for (const item of input) {
+    if (item !== undefined) {
+      result.push(item);
+    }
+  }
+  return result;
+}