webring 1.0.1 → 1.0.3

package/CHANGELOG.md

@@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.3](https://github.com/shepherdjerred/webring/compare/v1.0.2...v1.0.3) (2024-06-24)
+
+
+### Bug Fixes
+
+* create dir to cache ([a30645c](https://github.com/shepherdjerred/webring/commit/a30645c11d2afe91f7802c91b2c82eef9a97c717))
+
+## [1.0.2](https://github.com/shepherdjerred/webring/compare/v1.0.1...v1.0.2) (2024-06-21)
+
+
+### Bug Fixes
+
+* disable style parsing ([0999f02](https://github.com/shepherdjerred/webring/commit/0999f025b4e9970a20bde8c1ffce1248ce38f3b7))
+
 ## [1.0.1](https://github.com/shepherdjerred/webring/compare/v1.0.0...v1.0.1) (2024-06-03)
 
 

package/README.md

@@ -17,13 +17,13 @@ npm i webring
 
 ## Quick Start
 
-This library is meant to be used with static site generators. It is framework agnostic.
+This library is meant to be used with static site generators. It is framework agnostic. I use this with [Astro](https://astro.build/) on my [personal website](https://github.com/shepherdjerred/sjer.red/blob/main/src/components/BlogWebring.astro#L17-L22).
 
 ```typescript
-import { type Configuration, run } from "webring";
+import { run } from "webring";
+import { type Configuration, type Result } from "webring";
 
-// create a configuration object
-const config: Configuration = {
+export const config: Configuration = {
   sources: [
     {
       url: "https://drewdevault.com/blog/index.xml",
@@ -38,18 +38,39 @@ const config: Configuration = {
       title: "Jake Lazaroff",
     },
   ],
+  // the output will return the three most recent posts from the above sources
   number: 3,
-  cache_duration_minutes: 60,
+  // the output will return santized HTML truncated to 300 characters
   truncate: 300,
+  // if this is defined, we'll cache the results
+  cache: {
+    // the file to use as a cache
+    cache_file: "webring.json",
+    // how long the cache should remain valid for
+    cache_duration_minutes: 60,
+  },
 };
 
-// run the application
-const result = await run(config);
+// type Result = {
+//   title: string,
+//   url: string,
+//   date: Date,
+//   source: {
+//     url: string,
+//     title: string,
+//   },
+//   // this will be undefined if the RSS feed is empty
+//   preview?: string
+// }[]
+export const result: Result = await run(config);
 
-// do something with the results
 result.map((entry) => {
-  console.log(entry);
+  // do something with the results
+  // for example, you might render each item as an HTML block
 });
 ```
 
-I use this with Astro on my [personal website](https://github.com/shepherdjerred/sjer.red/blob/main/src/components/BlogWebring.astro#L17-L22).
+Here's what I do for my blog:
+
+1. Run `map` against the resulting array ([code](https://github.com/shepherdjerred/sjer.red/blob/f72b2b75bf0722ba8ff0fdd45f31c02c2ee5089d/src/components/BlogWebring.astro#L17-L22)).
+1. Render a component for each entry ([code](https://github.com/shepherdjerred/sjer.red/blob/f72b2b75bf0722ba8ff0fdd45f31c02c2ee5089d/src/components/WebringEntry.astro)).

package/dist/cache.d.ts

@@ -1,3 +1,3 @@
-import type { Configuration, Cache, Result, ResultEntry, Source } from "./types.js";
-export declare function runWithCache(config: Configuration, cache: Cache): Promise<[Result, Cache]>;
-export declare function fetchWithCache(source: Source, cache: Cache, config: Configuration): Promise<ResultEntry | undefined>;
+import { type Cache, type Result, type ResultEntry, type Source, type CachedConfiguration } from "./types.js";
+export declare function fetchAllCached(config: CachedConfiguration): Promise<Result>;
+export declare function fetchWithCache(source: Source, cache: Cache, config: CachedConfiguration): Promise<ResultEntry | undefined>;

package/dist/cache.js

@@ -1,25 +1,54 @@
 import * as R from "remeda";
+import { CacheSchema, } from "./types.js";
 import { fetch } from "./fetch.js";
-export async function runWithCache(config, cache) {
-    const promises = R.pipe(config.sources, R.map((source) => fetchWithCache(source, cache, config)));
-    const results = await Promise.all(promises);
-    const definedResults = results.filter((result) => result !== undefined);
-    const updatedCache = R.pipe(definedResults, R.map((result) => [result.source.url, { timestamp: new Date(), data: result }]), R.fromEntries());
-    const topResults = R.pipe(definedResults, R.sortBy((result) => result.date.getTime()), R.reverse(), R.take(config.number));
-    return [topResults, updatedCache];
+import fs from "fs/promises";
+import { asyncMapFilterUndefined } from "./util.js";
+async function loadCache({ cache_file }) {
+    try {
+        await fs.access(cache_file);
+        const cacheFile = await fs.readFile(cache_file);
+        return CacheSchema.parse(JSON.parse(cacheFile.toString()));
+    }
+    catch (e) {
+        return {};
+    }
+}
+async function saveCache({ cache_file }, cache) {
+    await fs.mkdir(cache_file.split("/").slice(0, -1).join("/"), { recursive: true });
+    await fs.writeFile(cache_file, JSON.stringify(cache));
+}
+function toCacheEntry(result, now) {
+    return [result.source.url, { timestamp: now, data: result }];
+}
+function toCache(results, now) {
+    return R.pipe(results, R.map((result) => toCacheEntry(result, now)), R.fromEntries());
+}
+function updateCache(results, config) {
+    const now = new Date();
+    const updatedCache = toCache(results, now);
+    return saveCache(config.cache, updatedCache);
+}
+export async function fetchAllCached(config) {
+    const cache = await loadCache(config.cache);
+    const results = await asyncMapFilterUndefined(config.sources, (source) => fetchWithCache(source, cache, config));
+    await updateCache(results, config);
+    return results;
 }
 export async function fetchWithCache(source, cache, config) {
     const cacheEntry = cache[source.url];
     if (cacheEntry) {
         const now = new Date();
-        if (now.getTime() - cacheEntry.timestamp.getTime() < config.cache_duration_minutes * 60 * 1000) {
-            console.log(`Cache entry found for ${source.url}`);
+        if (now.getTime() - cacheEntry.timestamp.getTime() < config.cache.cache_duration_minutes * 60 * 1000) {
+            console.log(`Cache entry found for ${source.url}.`);
             return Promise.resolve(cacheEntry.data);
         }
         else {
-            console.log(`Cache entry for ${source.url} is too old`);
+            console.log(`Cache entry for ${source.url} is too old.`);
         }
     }
-    console.log(`No cache entry for ${source.url}`);
+    else {
+        console.log(`No cache entry for ${source.url}.`);
+    }
+    console.log(`Fetching ${source.url}`);
     return fetch(source, config.truncate);
 }

package/dist/fetch.d.ts

@@ -1,2 +1,12 @@
-import { type Source, type ResultEntry } from "./types.js";
+import { type Source, type ResultEntry, type Configuration } from "./types.js";
+export declare function fetchAll(config: Configuration): Promise<{
+    url: string;
+    title: string;
+    date: Date;
+    source: {
+        url: string;
+        title: string;
+    };
+    preview?: string | undefined;
+}[]>;
 export declare function fetch(source: Source, length: number): Promise<ResultEntry | undefined>;

package/dist/fetch.js

@@ -3,8 +3,12 @@ import sanitizeHtml from "sanitize-html";
 import truncate from "truncate-html";
 import { FeedEntrySchema } from "./types.js";
 import * as R from "remeda";
+import { asyncMapFilterUndefined } from "./util.js";
 // for some reason, TypeScript does not infer the type of the default export correctly
 const truncateFn = truncate;
+export async function fetchAll(config) {
+    return await asyncMapFilterUndefined(config.sources, (source) => fetch(source, config.truncate));
+}
 export async function fetch(source, length) {
     const parser = new Parser();
     try {
@@ -19,7 +23,11 @@ export async function fetch(source, length) {
             url: firstItem.link,
             date: new Date(firstItem.date),
             source,
-            preview: preview ? truncateFn(sanitizeHtml(preview), length) : undefined,
+            preview: preview
+                ? truncateFn(sanitizeHtml(preview, {
+                    parseStyleAttributes: false,
+                }), length)
+                : undefined,
         };
     }
     catch (e) {

package/dist/index.d.ts

@@ -1,5 +1,3 @@
 import { type Configuration, type Result } from "./types.js";
 export declare function run(config: Configuration): Promise<Result>;
 export * from "./types.js";
-export * from "./cache.js";
-export * from "./fetch.js";

package/dist/index.js

@@ -1,29 +1,19 @@
-import { runWithCache } from "./cache.js";
-import { CacheSchema } from "./types.js";
-import fs from "fs/promises";
+import * as R from "remeda";
+import { fetchAllCached as fetchAllCached } from "./cache.js";
+import { fetchAll as fetchAllUncached } from "./fetch.js";
+import { CachedConfigurationSchema } from "./types.js";
 export async function run(config) {
-    const cacheFilename = config.cache_file;
-    let cacheObject = {};
-    // check if cache.json exists
-    try {
-        await fs.access(cacheFilename);
+    const { success, data } = CachedConfigurationSchema.safeParse(config);
+    let result;
+    if (success) {
+        console.log(`Using cache at ${data.cache.cache_file}.`);
+        result = await fetchAllCached(data);
     }
-    catch (e) {
-        await fs.writeFile(cacheFilename, JSON.stringify({}));
+    else {
+        console.log("Cache disabled.");
+        result = await fetchAllUncached(config);
     }
-    try {
-        const cacheFile = await fs.readFile(cacheFilename);
-        cacheObject = CacheSchema.parse(JSON.parse(cacheFile.toString()));
-    }
-    catch (e) {
-        console.error("Error reading cache file:", e);
-        throw e;
-    }
-    const [result, updatedCache] = await runWithCache(config, cacheObject);
-    // write the updated cache to cache.json
-    await fs.writeFile(cacheFilename, JSON.stringify(updatedCache));
-    return result;
+    const topResults = R.pipe(result, R.sortBy((result) => result.date.getTime()), R.reverse(), R.take(config.number));
+    return topResults;
 }
 export * from "./types.js";
-export * from "./cache.js";
-export * from "./fetch.js";

package/dist/index.test.js

@@ -1,7 +1,10 @@
-// sum.test.js
 import { expect, test } from "vitest";
 import { run } from "./index.js";
-test("it should work", async () => {
+import { tmpdir } from "os";
+import { mkdtemp } from "fs/promises";
+import { join } from "path";
+// TODO: intercept network requests
+test("it should fetch an RSS feed without caching", async () => {
     const config = {
         sources: [
             {
@@ -10,10 +13,94 @@ test("it should work", async () => {
             },
         ],
         number: 1,
-        cache_duration_minutes: 0,
         truncate: 300,
-        cache_file: "cache.json",
     };
     const result = await run(config);
     expect(result).toMatchSnapshot();
 });
+test("it should fetch several RSS feeds", async () => {
+    const config = {
+        sources: [
+            {
+                url: "https://drewdevault.com/blog/index.xml",
+                title: "Drew DeVault",
+            },
+            {
+                url: "https://danluu.com/atom.xml",
+                title: "Dan Luu",
+            },
+            {
+                url: "https://jakelazaroff.com/rss.xml",
+                title: "Jake Lazaroff",
+            },
+            {
+                url: "https://awesomekling.github.io/feed.xml",
+                title: "Andreas Kling",
+            },
+            {
+                url: "https://xeiaso.net/blog.rss",
+                title: "Xe Iaso",
+            },
+            {
+                url: "https://ciechanow.ski/atom.xml",
+                title: "Bartosz Ciechanowski",
+            },
+            {
+                url: "https://explained-from-first-principles.com/feed.xml",
+                title: "Explained From First Principles",
+            },
+            {
+                url: "http://www.aaronsw.com/2002/feeds/pgessays.rss",
+                title: "Paul Graham",
+            },
+            {
+                url: "https://samwho.dev/rss.xml",
+                title: "Sam Rose",
+            },
+            {
+                url: "https://rachelbythebay.com/w/atom.xml",
+                title: "Rachel Kroll",
+            },
+            {
+                url: "https://brr.fyi/feed.xml",
+                title: "brr.fyi",
+            },
+            {
+                url: "https://devblogs.microsoft.com/oldnewthing/feed",
+                title: "The Old New Thing",
+            },
+            {
+                url: "https://ludic.mataroa.blog/rss/",
+                title: "Ludicity",
+            },
+        ],
+        number: 20,
+        truncate: 300,
+    };
+    const result = await run(config);
+    expect(result).toMatchSnapshot();
+});
+test("it should fetch an RSS feed with caching", async () => {
+    const config = {
+        sources: [
+            {
+                title: "Jerred Shepherd",
+                url: "https://sjer.red/rss.xml",
+            },
+        ],
+        number: 1,
+        truncate: 300,
+        cache: {
+            cache_file: `${await createTempDir()}/cache.json`,
+            cache_duration_minutes: 1,
+        },
+    };
+    const result = await run(config);
+    expect(result).toMatchSnapshot();
+});
+// https://sdorra.dev/posts/2024-02-12-vitest-tmpdir
+async function createTempDir() {
+    const ostmpdir = tmpdir();
+    const dir = join(ostmpdir, "unit-test-");
+    return await mkdtemp(dir);
+}

package/dist/types.d.ts

@@ -10,6 +10,17 @@ declare const SourceSchema: z.ZodObject<{
     url: string;
     title: string;
 }>;
+export type CacheConfiguration = z.infer<typeof CacheConfigurationSchema>;
+declare const CacheConfigurationSchema: z.ZodObject<{
+    cache_duration_minutes: z.ZodDefault<z.ZodNumber>;
+    cache_file: z.ZodDefault<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    cache_duration_minutes: number;
+    cache_file: string;
+}, {
+    cache_duration_minutes?: number | undefined;
+    cache_file?: string | undefined;
+}>;
 export type Configuration = z.infer<typeof ConfigurationSchema>;
 declare const ConfigurationSchema: z.ZodObject<{
     sources: z.ZodArray<z.ZodObject<{
@@ -23,27 +34,97 @@ declare const ConfigurationSchema: z.ZodObject<{
         title: string;
     }>, "many">;
     number: z.ZodDefault<z.ZodNumber>;
-    cache_duration_minutes: z.ZodDefault<z.ZodNumber>;
     truncate: z.ZodDefault<z.ZodNumber>;
-    cache_file: z.ZodDefault<z.ZodString>;
+    cache: z.ZodOptional<z.ZodObject<{
+        cache_duration_minutes: z.ZodDefault<z.ZodNumber>;
+        cache_file: z.ZodDefault<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        cache_duration_minutes: number;
+        cache_file: string;
+    }, {
+        cache_duration_minutes?: number | undefined;
+        cache_file?: string | undefined;
+    }>>;
 }, "strip", z.ZodTypeAny, {
     number: number;
     sources: {
         url: string;
         title: string;
     }[];
-    cache_duration_minutes: number;
     truncate: number;
-    cache_file: string;
+    cache?: {
+        cache_duration_minutes: number;
+        cache_file: string;
+    } | undefined;
 }, {
     sources: {
         url: string;
         title: string;
     }[];
     number?: number | undefined;
-    cache_duration_minutes?: number | undefined;
     truncate?: number | undefined;
-    cache_file?: string | undefined;
+    cache?: {
+        cache_duration_minutes?: number | undefined;
+        cache_file?: string | undefined;
+    } | undefined;
+}>;
+export type CachedConfiguration = z.infer<typeof CachedConfigurationSchema>;
+export declare const CachedConfigurationSchema: z.ZodObject<z.objectUtil.extendShape<{
+    sources: z.ZodArray<z.ZodObject<{
+        url: z.ZodString;
+        title: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        url: string;
+        title: string;
+    }, {
+        url: string;
+        title: string;
+    }>, "many">;
+    number: z.ZodDefault<z.ZodNumber>;
+    truncate: z.ZodDefault<z.ZodNumber>;
+    cache: z.ZodOptional<z.ZodObject<{
+        cache_duration_minutes: z.ZodDefault<z.ZodNumber>;
+        cache_file: z.ZodDefault<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        cache_duration_minutes: number;
+        cache_file: string;
+    }, {
+        cache_duration_minutes?: number | undefined;
+        cache_file?: string | undefined;
+    }>>;
+}, {
+    cache: z.ZodObject<{
+        cache_duration_minutes: z.ZodDefault<z.ZodNumber>;
+        cache_file: z.ZodDefault<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        cache_duration_minutes: number;
+        cache_file: string;
+    }, {
+        cache_duration_minutes?: number | undefined;
+        cache_file?: string | undefined;
+    }>;
+}>, "strip", z.ZodTypeAny, {
+    number: number;
+    sources: {
+        url: string;
+        title: string;
+    }[];
+    truncate: number;
+    cache: {
+        cache_duration_minutes: number;
+        cache_file: string;
+    };
+}, {
+    sources: {
+        url: string;
+        title: string;
+    }[];
+    cache: {
+        cache_duration_minutes?: number | undefined;
+        cache_file?: string | undefined;
+    };
+    number?: number | undefined;
+    truncate?: number | undefined;
 }>;
 export type ResultEntry = z.infer<typeof ResultEntrySchema>;
 declare const ResultEntrySchema: z.ZodObject<{

package/dist/types.js

@@ -5,16 +5,22 @@ const SourceSchema = z.object({
     // a title for the feed
     title: z.string(),
 });
+const CacheConfigurationSchema = z.object({
+    // how long to cache a results for
+    cache_duration_minutes: z.number().default(60),
+    cache_file: z.string().default("cache.json"),
+});
 const ConfigurationSchema = z.object({
     // list of sources to fetch
     sources: SourceSchema.array(),
     // how many entries to return
     number: z.number().default(3),
-    // how long to cache a results for
-    cache_duration_minutes: z.number().default(60),
     // how many words to truncate the preview to
     truncate: z.number().default(300),
-    cache_file: z.string().default("cache.json"),
+    cache: CacheConfigurationSchema.optional(),
+});
+export const CachedConfigurationSchema = ConfigurationSchema.extend({
+    cache: CacheConfigurationSchema,
 });
 const ResultEntrySchema = z.object({
     title: z.string(),

package/dist/util.d.ts

@@ -0,0 +1,3 @@
+export declare function asyncMapFilterUndefined<T, U>(input: T[], fn: (x: T) => Promise<U | undefined>): Promise<U[]>;
+export declare function asyncMap<T, U>(input: T[], fn: (x: T) => Promise<U>): Promise<U[]>;
+export declare function filterUndefined<T>(input: (T | undefined)[]): T[];

package/dist/util.js

@@ -0,0 +1,13 @@
+import * as R from "remeda";
+// run an async map operation, filtering out undefined results
+export async function asyncMapFilterUndefined(input, fn) {
+    const results = await asyncMap(input, fn);
+    return filterUndefined(results);
+}
+export async function asyncMap(input, fn) {
+    const promises = R.pipe(input, R.map((item) => fn(item)));
+    return Promise.all(promises);
+}
+export function filterUndefined(input) {
+    return R.pipe(input, R.filter((result) => result !== undefined));
+}

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "webring",
   "type": "module",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "scripts": {
     "prepare": "husky",
     "lint": "eslint src",
     "build": "tsc",
-    "test": "vitest"
+    "test": "vitest --disable-console-intercept"
   },
   "main": "dist/index.js",
   "types": "dist/index.d.ts",