webring 0.3.0 → 1.0.1

package/CHANGELOG.md

@@ -5,13 +5,26 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [0.3.0](https://github.com/shepherdjerred/webring/compare/v0.2.0...v0.3.0) (2024-06-03)
+## [1.0.1](https://github.com/shepherdjerred/webring/compare/v1.0.0...v1.0.1) (2024-06-03)
 
 
-### Features
+### Bug Fixes
+
+* add tests, fix import ([3f811a1](https://github.com/shepherdjerred/webring/commit/3f811a18fae9186795f5d34cb0d4bbdd20f2a5df))
+
+## [1.0.0](https://github.com/shepherdjerred/webring/compare/v0.3.0...v1.0.0) (2024-06-03)
+
+
+### ⚠ BREAKING CHANGES
 
-* allow filename to be configurable ([cc7bb5f](https://github.com/shepherdjerred/webring/commit/cc7bb5f3139f306952d03568fc63cc9fcbfaad5e))
+* add link
 
-## [Unreleased]
+### Documentation
+
+* add link ([1a5d327](https://github.com/shepherdjerred/webring/commit/1a5d327a002785809e84aab70b19d18dd135f78b))
+
+## [0.3.0](https://github.com/shepherdjerred/webring/compare/v0.2.0...v0.3.0) (2024-06-03)
+
+### Features
 
-* Initial release
+- allow filename to be configurable ([cc7bb5f](https://github.com/shepherdjerred/webring/commit/cc7bb5f3139f306952d03568fc63cc9fcbfaad5e))

package/README.md

@@ -51,3 +51,5 @@ result.map((entry) => {
   console.log(entry);
 });
 ```
+
+I use this with Astro on my [personal website](https://github.com/shepherdjerred/sjer.red/blob/main/src/components/BlogWebring.astro#L17-L22).

package/dist/cache.d.ts

@@ -0,0 +1,3 @@
+import type { Configuration, Cache, Result, ResultEntry, Source } from "./types.js";
+export declare function runWithCache(config: Configuration, cache: Cache): Promise<[Result, Cache]>;
+export declare function fetchWithCache(source: Source, cache: Cache, config: Configuration): Promise<ResultEntry | undefined>;

package/dist/cache.js

@@ -0,0 +1,25 @@
+import * as R from "remeda";
+import { fetch } from "./fetch.js";
+export async function runWithCache(config, cache) {
+    const promises = R.pipe(config.sources, R.map((source) => fetchWithCache(source, cache, config)));
+    const results = await Promise.all(promises);
+    const definedResults = results.filter((result) => result !== undefined);
+    const updatedCache = R.pipe(definedResults, R.map((result) => [result.source.url, { timestamp: new Date(), data: result }]), R.fromEntries());
+    const topResults = R.pipe(definedResults, R.sortBy((result) => result.date.getTime()), R.reverse(), R.take(config.number));
+    return [topResults, updatedCache];
+}
+export async function fetchWithCache(source, cache, config) {
+    const cacheEntry = cache[source.url];
+    if (cacheEntry) {
+        const now = new Date();
+        if (now.getTime() - cacheEntry.timestamp.getTime() < config.cache_duration_minutes * 60 * 1000) {
+            console.log(`Cache entry found for ${source.url}`);
+            return Promise.resolve(cacheEntry.data);
+        }
+        else {
+            console.log(`Cache entry for ${source.url} is too old`);
+        }
+    }
+    console.log(`No cache entry for ${source.url}`);
+    return fetch(source, config.truncate);
+}

package/dist/fetch.d.ts

@@ -0,0 +1,2 @@
+import { type Source, type ResultEntry } from "./types.js";
+export declare function fetch(source: Source, length: number): Promise<ResultEntry | undefined>;

package/dist/fetch.js

@@ -0,0 +1,29 @@
+import Parser from "rss-parser";
+import sanitizeHtml from "sanitize-html";
+import truncate from "truncate-html";
+import { FeedEntrySchema } from "./types.js";
+import * as R from "remeda";
+// for some reason, TypeScript does not infer the type of the default export correctly
+const truncateFn = truncate;
+export async function fetch(source, length) {
+    const parser = new Parser();
+    try {
+        const feed = await parser.parseURL(source.url);
+        const firstItem = R.pipe(feed.items, R.map((item) => FeedEntrySchema.parse(item)), R.sortBy((item) => new Date(item.date).getTime()), R.reverse(), R.first());
+        if (!firstItem) {
+            throw new Error("no items found in feed");
+        }
+        const preview = firstItem.contentSnippet ?? firstItem.content ?? firstItem.description ?? firstItem["content:encoded"];
+        return {
+            title: firstItem.title,
+            url: firstItem.link,
+            date: new Date(firstItem.date),
+            source,
+            preview: preview ? truncateFn(sanitizeHtml(preview), length) : undefined,
+        };
+    }
+    catch (e) {
+        console.error(`Error fetching ${source.url}: ${e}`);
+        return undefined;
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+import { type Configuration, type Result } from "./types.js";
+export declare function run(config: Configuration): Promise<Result>;
+export * from "./types.js";
+export * from "./cache.js";
+export * from "./fetch.js";

package/dist/index.js

@@ -0,0 +1,29 @@
+import { runWithCache } from "./cache.js";
+import { CacheSchema } from "./types.js";
+import fs from "fs/promises";
+export async function run(config) {
+    const cacheFilename = config.cache_file;
+    let cacheObject = {};
+    // check if cache.json exists
+    try {
+        await fs.access(cacheFilename);
+    }
+    catch (e) {
+        await fs.writeFile(cacheFilename, JSON.stringify({}));
+    }
+    try {
+        const cacheFile = await fs.readFile(cacheFilename);
+        cacheObject = CacheSchema.parse(JSON.parse(cacheFile.toString()));
+    }
+    catch (e) {
+        console.error("Error reading cache file:", e);
+        throw e;
+    }
+    const [result, updatedCache] = await runWithCache(config, cacheObject);
+    // write the updated cache to cache.json
+    await fs.writeFile(cacheFilename, JSON.stringify(updatedCache));
+    return result;
+}
+export * from "./types.js";
+export * from "./cache.js";
+export * from "./fetch.js";

package/dist/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/index.test.js

@@ -0,0 +1,19 @@
+// sum.test.js
+import { expect, test } from "vitest";
+import { run } from "./index.js";
+test("it should work", async () => {
+    const config = {
+        sources: [
+            {
+                title: "Jerred Shepherd",
+                url: "https://sjer.red/rss.xml",
+            },
+        ],
+        number: 1,
+        cache_duration_minutes: 0,
+        truncate: 300,
+        cache_file: "cache.json",
+    };
+    const result = await run(config);
+    expect(result).toMatchSnapshot();
+});

package/dist/types.d.ts

@@ -0,0 +1,287 @@
+import { z } from "zod";
+export type Source = z.infer<typeof SourceSchema>;
+declare const SourceSchema: z.ZodObject<{
+    url: z.ZodString;
+    title: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    title: string;
+}, {
+    url: string;
+    title: string;
+}>;
+export type Configuration = z.infer<typeof ConfigurationSchema>;
+declare const ConfigurationSchema: z.ZodObject<{
+    sources: z.ZodArray<z.ZodObject<{
+        url: z.ZodString;
+        title: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        url: string;
+        title: string;
+    }, {
+        url: string;
+        title: string;
+    }>, "many">;
+    number: z.ZodDefault<z.ZodNumber>;
+    cache_duration_minutes: z.ZodDefault<z.ZodNumber>;
+    truncate: z.ZodDefault<z.ZodNumber>;
+    cache_file: z.ZodDefault<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    number: number;
+    sources: {
+        url: string;
+        title: string;
+    }[];
+    cache_duration_minutes: number;
+    truncate: number;
+    cache_file: string;
+}, {
+    sources: {
+        url: string;
+        title: string;
+    }[];
+    number?: number | undefined;
+    cache_duration_minutes?: number | undefined;
+    truncate?: number | undefined;
+    cache_file?: string | undefined;
+}>;
+export type ResultEntry = z.infer<typeof ResultEntrySchema>;
+declare const ResultEntrySchema: z.ZodObject<{
+    title: z.ZodString;
+    url: z.ZodString;
+    date: z.ZodDate;
+    source: z.ZodObject<{
+        url: z.ZodString;
+        title: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        url: string;
+        title: string;
+    }, {
+        url: string;
+        title: string;
+    }>;
+    preview: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    title: string;
+    date: Date;
+    source: {
+        url: string;
+        title: string;
+    };
+    preview?: string | undefined;
+}, {
+    url: string;
+    title: string;
+    date: Date;
+    source: {
+        url: string;
+        title: string;
+    };
+    preview?: string | undefined;
+}>;
+export type Result = z.infer<typeof ResultSchema>;
+declare const ResultSchema: z.ZodArray<z.ZodObject<{
+    title: z.ZodString;
+    url: z.ZodString;
+    date: z.ZodDate;
+    source: z.ZodObject<{
+        url: z.ZodString;
+        title: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        url: string;
+        title: string;
+    }, {
+        url: string;
+        title: string;
+    }>;
+    preview: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    title: string;
+    date: Date;
+    source: {
+        url: string;
+        title: string;
+    };
+    preview?: string | undefined;
+}, {
+    url: string;
+    title: string;
+    date: Date;
+    source: {
+        url: string;
+        title: string;
+    };
+    preview?: string | undefined;
+}>, "many">;
+export type CacheEntry = z.infer<typeof CacheEntrySchema>;
+export declare const CacheEntrySchema: z.ZodObject<{
+    timestamp: z.ZodDate;
+    data: z.ZodObject<{
+        title: z.ZodString;
+        url: z.ZodString;
+        date: z.ZodDate;
+        source: z.ZodObject<{
+            url: z.ZodString;
+            title: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            url: string;
+            title: string;
+        }, {
+            url: string;
+            title: string;
+        }>;
+        preview: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    }, {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    timestamp: Date;
+    data: {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    };
+}, {
+    timestamp: Date;
+    data: {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    };
+}>;
+export type Cache = z.infer<typeof CacheSchema>;
+export declare const CacheSchema: z.ZodRecord<z.ZodString, z.ZodObject<{
+    timestamp: z.ZodDate;
+    data: z.ZodObject<{
+        title: z.ZodString;
+        url: z.ZodString;
+        date: z.ZodDate;
+        source: z.ZodObject<{
+            url: z.ZodString;
+            title: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            url: string;
+            title: string;
+        }, {
+            url: string;
+            title: string;
+        }>;
+        preview: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    }, {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    timestamp: Date;
+    data: {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    };
+}, {
+    timestamp: Date;
+    data: {
+        url: string;
+        title: string;
+        date: Date;
+        source: {
+            url: string;
+            title: string;
+        };
+        preview?: string | undefined;
+    };
+}>>;
+export declare const FeedEntrySchema: z.ZodEffects<z.ZodObject<{
+    title: z.ZodString;
+    link: z.ZodString;
+    isoDate: z.ZodOptional<z.ZodDate>;
+    pubDate: z.ZodOptional<z.ZodDate>;
+    content: z.ZodOptional<z.ZodString>;
+    contentSnippet: z.ZodOptional<z.ZodString>;
+    "content:encoded": z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    title: string;
+    link: string;
+    isoDate?: Date | undefined;
+    pubDate?: Date | undefined;
+    content?: string | undefined;
+    contentSnippet?: string | undefined;
+    "content:encoded"?: string | undefined;
+    description?: string | undefined;
+}, {
+    title: string;
+    link: string;
+    isoDate?: Date | undefined;
+    pubDate?: Date | undefined;
+    content?: string | undefined;
+    contentSnippet?: string | undefined;
+    "content:encoded"?: string | undefined;
+    description?: string | undefined;
+}>, {
+    title: string;
+    link: string;
+    date: Date;
+    content: string | undefined;
+    contentSnippet: string | undefined;
+    description: string | undefined;
+    "content:encoded": string | undefined;
+}, {
+    title: string;
+    link: string;
+    isoDate?: Date | undefined;
+    pubDate?: Date | undefined;
+    content?: string | undefined;
+    contentSnippet?: string | undefined;
+    "content:encoded"?: string | undefined;
+    description?: string | undefined;
+}>;
+export {};

package/dist/types.js

@@ -0,0 +1,57 @@
+import { z } from "zod";
+const SourceSchema = z.object({
+    // the url of the feed
+    url: z.string(),
+    // a title for the feed
+    title: z.string(),
+});
+const ConfigurationSchema = z.object({
+    // list of sources to fetch
+    sources: SourceSchema.array(),
+    // how many entries to return
+    number: z.number().default(3),
+    // how long to cache a results for
+    cache_duration_minutes: z.number().default(60),
+    // how many words to truncate the preview to
+    truncate: z.number().default(300),
+    cache_file: z.string().default("cache.json"),
+});
+const ResultEntrySchema = z.object({
+    title: z.string(),
+    url: z.string(),
+    date: z.coerce.date(),
+    source: SourceSchema,
+    preview: z.string().optional(),
+});
+const ResultSchema = z.array(ResultEntrySchema);
+export const CacheEntrySchema = z.object({
+    timestamp: z.coerce.date(),
+    data: ResultEntrySchema,
+});
+export const CacheSchema = z.record(CacheEntrySchema);
+export const FeedEntrySchema = z
+    .object({
+    title: z.string(),
+    link: z.string(),
+    isoDate: z.coerce.date().optional(),
+    pubDate: z.coerce.date().optional(),
+    content: z.string().optional(),
+    contentSnippet: z.string().optional(),
+    "content:encoded": z.string().optional(),
+    description: z.string().optional(),
+})
+    .transform((entry) => {
+    const date = entry.isoDate ?? entry.pubDate;
+    if (!date) {
+        throw new Error("no date found in feed entry");
+    }
+    return {
+        title: entry.title,
+        link: entry.link,
+        date,
+        content: entry.content,
+        contentSnippet: entry.contentSnippet,
+        description: entry.description,
+        "content:encoded": entry["content:encoded"],
+    };
+});

package/package.json

@@ -1,17 +1,17 @@
 {
   "name": "webring",
   "type": "module",
-  "version": "0.3.0",
+  "version": "1.0.1",
   "scripts": {
-    "lint": "eslint .",
+    "prepare": "husky",
+    "lint": "eslint src",
     "build": "tsc",
-    "test": "",
-    "prepare": "husky"
+    "test": "vitest"
   },
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "dependencies": {
-    "remeda": "^2.0.0",
+    "remeda": "^2.0.1",
     "rss-parser": "^3.13.0",
     "sanitize-html": "^2.13.0",
     "truncate-html": "^1.1.1",
@@ -24,21 +24,30 @@
     "@tsconfig/node20": "^20.1.4",
     "@tsconfig/strictest": "^2.0.5",
     "@types/eslint__js": "^8.42.3",
-    "@types/node": "^20.13.0",
+    "@types/node": "^20.14.0",
     "@types/sanitize-html": "^2.11.0",
     "@typescript-eslint/eslint-plugin": "^7.11.0",
     "@typescript-eslint/parser": "^7.11.0",
     "eslint": "^8.57.0",
     "husky": "^9.0.11",
     "lint-staged": "^15.2.5",
-    "prettier": "^3.2.5",
+    "prettier": "^3.3.0",
     "typescript": "^5.4.5",
-    "typescript-eslint": "^7.11.0"
+    "typescript-eslint": "^7.11.0",
+    "vitest": "^1.6.0"
   },
   "lint-staged": {
     "*.{ts,tsx}": "eslint --cache --fix",
     "*": "prettier --ignore-unknown --write"
   },
+  "commitlint": {
+    "extends": [
+      "@commitlint/config-conventional"
+    ]
+  },
+  "prettier": {
+    "printWidth": 120
+  },
   "files": [
     "dist",
     "package.json",