webring 1.5.1 → 1.6.0-dev.893
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +104 -135
- package/bun.lock +565 -305
- package/dist/cache.d.ts +1 -1
- package/dist/cache.d.ts.map +1 -1
- package/dist/cache.js +19 -16
- package/dist/cache.js.map +1 -1
- package/dist/fetch.d.ts +1 -1
- package/dist/fetch.d.ts.map +1 -1
- package/dist/fetch.js +28 -7
- package/dist/fetch.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -14
- package/dist/index.js.map +1 -1
- package/dist/index.test.js +14 -14
- package/dist/index.test.js.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +1 -1
- package/dist/types.js.map +1 -1
- package/dist/util.d.ts.map +1 -1
- package/dist/util.js +7 -1
- package/dist/util.js.map +1 -1
- package/package.json +19 -28
- package/src/__snapshots__/index.test.ts.snap +7 -0
- package/src/cache.ts +39 -22
- package/src/fetch.ts +46 -10
- package/src/index.test.ts +16 -16
- package/src/index.ts +25 -17
- package/src/types.ts +1 -1
- package/src/util.ts +15 -6
- package/LICENSE +0 -674
package/dist/cache.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Cache, type Result, type ResultEntry, type Source, type CachedConfiguration } from "./types.
|
|
1
|
+
import { type Cache, type Result, type ResultEntry, type Source, type CachedConfiguration } from "./types.ts";
|
|
2
2
|
export declare function fetchAllCached(config: CachedConfiguration): Promise<Result>;
|
|
3
3
|
export declare function fetchWithCache(source: Source, cache: Cache, config: CachedConfiguration): Promise<ResultEntry | undefined>;
|
|
4
4
|
//# sourceMappingURL=cache.d.ts.map
|
package/dist/cache.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AACA,OAAO,EACL,KAAK,KAAK,EACV,KAAK,MAAM,EACX,KAAK,WAAW,EAEhB,KAAK,MAAM,EACX,KAAK,mBAAmB,EAGzB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AACA,OAAO,EACL,KAAK,KAAK,EACV,KAAK,MAAM,EACX,KAAK,WAAW,EAEhB,KAAK,MAAM,EACX,KAAK,mBAAmB,EAGzB,MAAM,YAAY,CAAC;AAmDpB,wBAAsB,cAAc,CAClC,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,MAAM,CAAC,CAUjB;AAED,wBAAsB,cAAc,CAClC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAmBlC"}
|
package/dist/cache.js
CHANGED
|
@@ -2,23 +2,25 @@ import * as R from "remeda";
|
|
|
2
2
|
import { CacheSchema, } from "./types.js";
|
|
3
3
|
import { fetch } from "./fetch.js";
|
|
4
4
|
import { asyncMapFilterUndefined } from "./util.js";
|
|
5
|
-
import * as fs from "fs/promises";
|
|
6
|
-
async function loadCache({ cache_file }) {
|
|
5
|
+
import * as fs from "node:fs/promises";
|
|
6
|
+
async function loadCache({ cache_file: cacheFilePath, }) {
|
|
7
7
|
try {
|
|
8
|
-
await fs.access(
|
|
9
|
-
const
|
|
10
|
-
return CacheSchema.parse(JSON.parse(
|
|
8
|
+
await fs.access(cacheFilePath);
|
|
9
|
+
const cacheFileContent = await fs.readFile(cacheFilePath);
|
|
10
|
+
return CacheSchema.parse(JSON.parse(cacheFileContent.toString()));
|
|
11
11
|
}
|
|
12
|
-
catch
|
|
12
|
+
catch {
|
|
13
13
|
return {};
|
|
14
14
|
}
|
|
15
15
|
}
|
|
16
|
-
async function saveCache({ cache_file }, cache) {
|
|
17
|
-
const dir =
|
|
16
|
+
async function saveCache({ cache_file: cacheFilePath }, cache) {
|
|
17
|
+
const dir = cacheFilePath.split("/").slice(0, -1).join("/");
|
|
18
18
|
if (dir !== "") {
|
|
19
|
-
await fs.mkdir(
|
|
19
|
+
await fs.mkdir(cacheFilePath.split("/").slice(0, -1).join("/"), {
|
|
20
|
+
recursive: true,
|
|
21
|
+
});
|
|
20
22
|
}
|
|
21
|
-
await fs.writeFile(
|
|
23
|
+
await fs.writeFile(cacheFilePath, JSON.stringify(cache));
|
|
22
24
|
}
|
|
23
25
|
function toCacheEntry(result, now) {
|
|
24
26
|
return [result.source.url, { timestamp: now, data: result }];
|
|
@@ -41,18 +43,19 @@ export async function fetchWithCache(source, cache, config) {
|
|
|
41
43
|
const cacheEntry = cache[source.url];
|
|
42
44
|
if (cacheEntry) {
|
|
43
45
|
const now = new Date();
|
|
44
|
-
if (now.getTime() - cacheEntry.timestamp.getTime() <
|
|
45
|
-
|
|
46
|
-
|
|
46
|
+
if (now.getTime() - cacheEntry.timestamp.getTime() <
|
|
47
|
+
config.cache.cache_duration_minutes * 60 * 1000) {
|
|
48
|
+
console.warn(`Cache entry found for ${source.url}.`);
|
|
49
|
+
return cacheEntry.data;
|
|
47
50
|
}
|
|
48
51
|
else {
|
|
49
|
-
console.
|
|
52
|
+
console.warn(`Cache entry for ${source.url} is too old.`);
|
|
50
53
|
}
|
|
51
54
|
}
|
|
52
55
|
else {
|
|
53
|
-
console.
|
|
56
|
+
console.warn(`No cache entry for ${source.url}.`);
|
|
54
57
|
}
|
|
55
|
-
console.
|
|
58
|
+
console.warn(`Fetching ${source.url}`);
|
|
56
59
|
return fetch(source, config.truncate);
|
|
57
60
|
}
|
|
58
61
|
//# sourceMappingURL=cache.js.map
|
package/dist/cache.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAOL,WAAW,GAEZ,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAOL,WAAW,GAEZ,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEvC,KAAK,UAAU,SAAS,CAAC,EACvB,UAAU,EAAE,aAAa,GACN;IACnB,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAC/B,MAAM,gBAAgB,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC1D,OAAO,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,EAAE,UAAU,EAAE,aAAa,EAAsB,EACjD,KAAY;IAEZ,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5D,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YAC9D,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;IACL,CAAC;IACD,MAAM,EAAE,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,YAAY,CAAC,MAAmB,EAAE,GAAS;IAClD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,OAAO,CAAC,OAAsB,EAAE,GAAS;IAChD,OAAO,CAAC,CAAC,IAAI,CACX,OAAO,EACP,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,EAC5C,CAAC,CAAC,WAAW,EAAE,CAChB,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,OAAsB,EACtB,MAA2B;IAE3B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC3C,OAAO,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAA2B;IAE3B,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAE5C,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CACvE,cAAc,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CACtC,CAAC;IAEF,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAEnC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,KAAY,EACZ,MAA2B;IAE3B,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IACE,GAAG,CAAC,OAAO,EAAE,GAAG,UAAU,CAAC,SAAS,CAAC,OAAO,EAAE;YAC9C,MAAM,CAAC,KAAK,CAAC,sBAAsB,GAAG,EAAE,GAAG,IAAI,EAC/C,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;YACrD,OAAO,UAAU,CAAC,IAAI,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,sBAAsB,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IACvC,OAAO,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;AACxC,CAAC"}
|
package/dist/fetch.d.ts
CHANGED
package/dist/fetch.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,MAAM,EACX,KAAK,WAAW,EAEhB,KAAK,aAAa,EACnB,MAAM,YAAY,CAAC;AAwBpB,wBAAsB,QAAQ,CAAC,MAAM,EAAE,aAAa;;;;;;;;;;KAInD;AAED,wBAAsB,KAAK,CACzB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAyClC"}
|
package/dist/fetch.js
CHANGED
|
@@ -1,12 +1,28 @@
|
|
|
1
1
|
import Parser from "rss-parser";
|
|
2
2
|
import sanitizeHtml from "sanitize-html";
|
|
3
3
|
import * as truncateHtml from "truncate-html";
|
|
4
|
-
import {
|
|
4
|
+
import { z } from "zod/v3";
|
|
5
|
+
import { FeedEntrySchema, } from "./types.js";
|
|
5
6
|
import * as R from "remeda";
|
|
6
7
|
import { asyncMapFilterUndefined } from "./util.js";
|
|
7
8
|
// Handle ESM/CommonJS interop - truncate-html exports differently in different environments
|
|
8
|
-
|
|
9
|
-
|
|
9
|
+
const TruncateFnSchema = z
|
|
10
|
+
.function()
|
|
11
|
+
.args(z.string(), z.number().optional())
|
|
12
|
+
.returns(z.string());
|
|
13
|
+
const TruncateModuleSchema = z.object({ default: TruncateFnSchema });
|
|
14
|
+
function truncate(html, length) {
|
|
15
|
+
const mod = truncateHtml;
|
|
16
|
+
const fnResult = TruncateFnSchema.safeParse(mod);
|
|
17
|
+
if (fnResult.success) {
|
|
18
|
+
return fnResult.data(html, length);
|
|
19
|
+
}
|
|
20
|
+
const modResult = TruncateModuleSchema.safeParse(mod);
|
|
21
|
+
if (modResult.success) {
|
|
22
|
+
return modResult.data.default(html, length);
|
|
23
|
+
}
|
|
24
|
+
throw new Error("truncate-html module could not be resolved");
|
|
25
|
+
}
|
|
10
26
|
export async function fetchAll(config) {
|
|
11
27
|
return await asyncMapFilterUndefined(config.sources, (source) => fetch(source, config.truncate));
|
|
12
28
|
}
|
|
@@ -18,17 +34,22 @@ export async function fetch(source, length) {
|
|
|
18
34
|
if (!firstItem) {
|
|
19
35
|
throw new Error("no items found in feed");
|
|
20
36
|
}
|
|
21
|
-
const preview = firstItem.contentSnippet ??
|
|
37
|
+
const preview = firstItem.contentSnippet ??
|
|
38
|
+
firstItem.content ??
|
|
39
|
+
firstItem.description ??
|
|
40
|
+
firstItem["content:encoded"];
|
|
22
41
|
return {
|
|
23
42
|
title: firstItem.title,
|
|
24
43
|
url: firstItem.link,
|
|
25
44
|
date: new Date(firstItem.date),
|
|
26
45
|
source,
|
|
27
|
-
preview: preview
|
|
46
|
+
preview: preview !== undefined && preview !== ""
|
|
47
|
+
? truncate(sanitizeHtml(preview, { parseStyleAttributes: false }), length)
|
|
48
|
+
: undefined,
|
|
28
49
|
};
|
|
29
50
|
}
|
|
30
|
-
catch (
|
|
31
|
-
console.error(`Error fetching ${source.url}: ${
|
|
51
|
+
catch (error) {
|
|
52
|
+
console.error(`Error fetching ${source.url}: ${String(error)}`);
|
|
32
53
|
return undefined;
|
|
33
54
|
}
|
|
34
55
|
}
|
package/dist/fetch.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,YAAY,CAAC;AAChC,OAAO,YAAY,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,YAAY,MAAM,eAAe,CAAC;AAC9C,OAAO,
|
|
1
|
+
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,YAAY,CAAC;AAChC,OAAO,YAAY,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,YAAY,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAC3B,OAAO,EAGL,eAAe,GAEhB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,CAAC,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAC;AAEpD,4FAA4F;AAC5F,MAAM,gBAAgB,GAAG,CAAC;KACvB,QAAQ,EAAE;KACV,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC;KACvC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;AACvB,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;AAErE,SAAS,QAAQ,CAAC,IAAY,EAAE,MAAe;IAC7C,MAAM,GAAG,GAAG,YAAuB,CAAC;IACpC,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjD,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,SAAS,GAAG,oBAAoB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACtD,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAqB;IAClD,OAAO,MAAM,uBAAuB,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAC9D,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,CAC/B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,MAAc,EACd,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAE5B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CACtB,IAAI,CAAC,KAAK,EACV,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAC5C,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,EACjD,CAAC,CAAC,OAAO,EAAE,EACX,CAAC,CAAC,KAAK,EAAE,CACV,CAAC;QAEF,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,OAAO,GACX,SAAS,CAAC,cAAc;YACxB,SAAS,CAAC,OAAO;YACjB,SAAS,CAAC,WAAW;YACrB,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAE/B,OAAO;YACL,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,GAAG,EAAE,SAAS,CAAC,IAAI;YACnB,IAAI,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC9B,MAAM;YACN,OAAO,EACL,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE;gBACrC,CAAC,CAAC,QAAQ,CACN,YAAY,CAAC,OAAO,EAAE,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC,EACtD,MAAM,CACP;gBACH,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,kBAAkB,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChE,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Configuration, type Result } from "./types.
|
|
1
|
+
import { type Configuration, type Result } from "./types.ts";
|
|
2
|
+
export type { Configuration, Result, ResultEntry } from "./types.ts";
|
|
2
3
|
export declare function run(config: Configuration): Promise<Result>;
|
|
3
|
-
export * from "./types.js";
|
|
4
4
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,MAAM,EAEZ,MAAM,YAAY,CAAC;AAEpB,YAAY,EAAE,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAErE,wBAAsB,GAAG,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAsChE"}
|
package/dist/index.js
CHANGED
|
@@ -1,33 +1,33 @@
|
|
|
1
1
|
import * as R from "remeda";
|
|
2
|
-
import { fetchAllCached
|
|
2
|
+
import { fetchAllCached } from "./cache.js";
|
|
3
3
|
import { fetchAll as fetchAllUncached } from "./fetch.js";
|
|
4
|
-
import { CachedConfigurationSchema } from "./types.js";
|
|
4
|
+
import { CachedConfigurationSchema, } from "./types.js";
|
|
5
5
|
export async function run(config) {
|
|
6
6
|
const { success, data } = CachedConfigurationSchema.safeParse(config);
|
|
7
|
-
let
|
|
7
|
+
let fetched;
|
|
8
8
|
if (success) {
|
|
9
|
-
console.
|
|
10
|
-
|
|
9
|
+
console.warn(`Using cache at ${data.cache.cache_file}.`);
|
|
10
|
+
fetched = await fetchAllCached(data);
|
|
11
11
|
}
|
|
12
12
|
else {
|
|
13
|
-
console.
|
|
14
|
-
|
|
13
|
+
console.warn("Cache disabled.");
|
|
14
|
+
fetched = await fetchAllUncached(config);
|
|
15
15
|
}
|
|
16
|
-
let results = R.pipe(
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
16
|
+
let results = R.pipe(fetched, R.sortBy((entry) => entry.date.getTime()), R.reverse(), R.filter((entry) => {
|
|
17
|
+
const filterFn = entry.source.filter;
|
|
18
|
+
if (filterFn === undefined ||
|
|
19
|
+
entry.preview === undefined ||
|
|
20
|
+
entry.preview === "") {
|
|
21
21
|
return true;
|
|
22
22
|
}
|
|
23
|
+
return filterFn(entry.preview);
|
|
23
24
|
}));
|
|
24
25
|
// shuffle if wanted
|
|
25
|
-
if (config.shuffle) {
|
|
26
|
+
if (config.shuffle === true) {
|
|
26
27
|
results = R.shuffle(results);
|
|
27
28
|
}
|
|
28
29
|
// take n
|
|
29
30
|
results = R.take(results, config.number);
|
|
30
31
|
return results;
|
|
31
32
|
}
|
|
32
|
-
export * from "./types.js";
|
|
33
33
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,cAAc,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,QAAQ,IAAI,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAGL,yBAAyB,GAC1B,MAAM,YAAY,CAAC;AAIpB,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,MAAqB;IAC7C,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,yBAAyB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAEtE,IAAI,OAAe,CAAC;IACpB,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC;QACzD,OAAO,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAChC,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,IAAI,CAClB,OAAO,EACP,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EACzC,CAAC,CAAC,OAAO,EAAE,EACX,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QACjB,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;QACrC,IACE,QAAQ,KAAK,SAAS;YACtB,KAAK,CAAC,OAAO,KAAK,SAAS;YAC3B,KAAK,CAAC,OAAO,KAAK,EAAE,EACpB,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC,CAAC,CACH,CAAC;IAEF,oBAAoB;IACpB,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QAC5B,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAED,SAAS;IACT,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAEzC,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/index.test.js
CHANGED
|
@@ -1,26 +1,26 @@
|
|
|
1
|
-
import { expect, test } from "
|
|
1
|
+
import { expect, test } from "bun:test";
|
|
2
2
|
import { run } from "./index.js";
|
|
3
|
-
import { tmpdir } from "os";
|
|
4
|
-
import { mkdtemp } from "fs/promises";
|
|
5
|
-
import
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { mkdtemp } from "node:fs/promises";
|
|
5
|
+
import path from "node:path";
|
|
6
6
|
import express from "express";
|
|
7
7
|
const app = express();
|
|
8
8
|
app.use(express.static("src/testdata"));
|
|
9
|
-
const port = Math.floor(Math.random() *
|
|
9
|
+
const port = Math.floor(Math.random() * 10_000) + 3000;
|
|
10
10
|
app.listen(port, () => {
|
|
11
|
-
console.
|
|
11
|
+
console.warn(`Test server listening at http://localhost:${port.toString()}`);
|
|
12
12
|
});
|
|
13
13
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
14
|
-
function createUrl(
|
|
15
|
-
return `http://localhost:${port.toString()}/${
|
|
14
|
+
function createUrl(urlPath) {
|
|
15
|
+
return `http://localhost:${port.toString()}/${urlPath}`;
|
|
16
16
|
}
|
|
17
17
|
function createSources(count) {
|
|
18
18
|
return Array.from({ length: count }, (_, i) => ({
|
|
19
|
-
title: `rss ${i.toString()}`,
|
|
20
|
-
url: createUrl(`rss-${i.toString()}.xml`),
|
|
19
|
+
title: `rss ${(i + 1).toString()}`,
|
|
20
|
+
url: createUrl(`rss-${(i + 1).toString()}.xml`),
|
|
21
21
|
}));
|
|
22
22
|
}
|
|
23
|
-
test("it should fetch an RSS feed without caching",
|
|
23
|
+
test("it should fetch an RSS feed without caching", async () => {
|
|
24
24
|
const config = {
|
|
25
25
|
sources: createSources(19),
|
|
26
26
|
number: 1,
|
|
@@ -32,7 +32,7 @@ test("it should fetch an RSS feed without caching", { timeout: 30000 }, async ()
|
|
|
32
32
|
string = string.replaceAll(port.toString(), "PORT");
|
|
33
33
|
expect(string).toMatchSnapshot();
|
|
34
34
|
});
|
|
35
|
-
test("it should fetch several RSS feeds",
|
|
35
|
+
test("it should fetch several RSS feeds", async () => {
|
|
36
36
|
const config = {
|
|
37
37
|
sources: createSources(19),
|
|
38
38
|
number: 3,
|
|
@@ -44,7 +44,7 @@ test("it should fetch several RSS feeds", { timeout: 30000 }, async () => {
|
|
|
44
44
|
string = string.replaceAll(port.toString(), "PORT");
|
|
45
45
|
expect(string).toMatchSnapshot();
|
|
46
46
|
});
|
|
47
|
-
test("it should fetch an RSS feed with caching",
|
|
47
|
+
test("it should fetch an RSS feed with caching", async () => {
|
|
48
48
|
const config = {
|
|
49
49
|
sources: createSources(19),
|
|
50
50
|
number: 1,
|
|
@@ -63,7 +63,7 @@ test("it should fetch an RSS feed with caching", { timeout: 30000 }, async () =>
|
|
|
63
63
|
// https://sdorra.dev/posts/2024-02-12-vitest-tmpdir
|
|
64
64
|
async function createTempDir() {
|
|
65
65
|
const ostmpdir = tmpdir();
|
|
66
|
-
const dir = join(ostmpdir, "unit-test-");
|
|
66
|
+
const dir = path.join(ostmpdir, "unit-test-");
|
|
67
67
|
return await mkdtemp(dir);
|
|
68
68
|
}
|
|
69
69
|
//# sourceMappingURL=index.test.js.map
|
package/dist/index.test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../src/index.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../src/index.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAExC,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,MAAM,GAAG,GAAG,OAAO,EAAE,CAAC;AACtB,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;AAExC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;AACvD,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;IACpB,OAAO,CAAC,IAAI,CAAC,6CAA6C,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC,CAAC,CAAC;AAEH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;AAEzD,SAAS,SAAS,CAAC,OAAe;IAChC,OAAO,oBAAoB,IAAI,CAAC,QAAQ,EAAE,IAAI,OAAO,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9C,KAAK,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE;QAClC,GAAG,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC;KAChD,CAAC,CAAC,CAAC;AACN,CAAC;AAED,IAAI,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;IAC7D,MAAM,MAAM,GAAkB;QAC5B,OAAO,EAAE,aAAa,CAAC,EAAE,CAAC;QAC1B,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,GAAG;KACd,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpC,6CAA6C;IAC7C,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,EAAE,CAAC;AACnC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;IACnD,MAAM,MAAM,GAAkB;QAC5B,OAAO,EAAE,aAAa,CAAC,EAAE,CAAC;QAC1B,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,GAAG;KACd,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpC,6CAA6C;IAC7C,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,EAAE,CAAC;AACnC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;IAC1D,MAAM,MAAM,GAAkB;QAC5B,OAAO,EAAE,aAAa,CAAC,EAAE,CAAC;QAC1B,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,GAAG;QACb,KAAK,EAAE;YACL,UAAU,EAAE,GAAG,MAAM,aAAa,EAAE,aAAa;YACjD,sBAAsB,EAAE,CAAC;SAC1B;KACF,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpC,6CAA6C;IAC7C,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,EAAE,CAAC;AACnC,CAAC,CAAC,CAAC;AAEH,oDAAoD;AACpD,KAAK,UAAU,aAAa;IAC1B,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC;IAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC9C,OAAO,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;AAC5B,CAAC"}
|
package/dist/types.d.ts
CHANGED
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAE3B,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAClD,oBAAoB;AACpB,QAAA,MAAM,YAAY;IAChB,6BAA6B;;IAE7B,mCAAmC;;IAEnC,8EAA8E;;;;;;;;;;EAE9E,CAAC;AAEH,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,kCAAkC;AAClC,QAAA,MAAM,wBAAwB;IAC5B,qCAAqC;;IAErC,+CAA+C;;;;;;;;EAE/C,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,8DAA8D;AAC9D,QAAA,MAAM,mBAAmB;IACvB,iCAAiC;;QApBjC,6BAA6B;;QAE7B,mCAAmC;;QAEnC,8EAA8E;;;;;;;;;;;IAkB9E,wDAAwD;;IAExD,sHAAsH;;IAEtH,kCAAkC;;QAflC,qCAAqC;;QAErC,+CAA+C;;;;;;;;;IAe/C,iCAAiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAEjC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,gEAAgE;AAChE,eAAO,MAAM,yBAAyB;;QAlCpC,6BAA6B;;QAE7B,mCAAmC;;QAEnC,8EAA8E;;;;;;;;;;;;;;;;QAO9E,qCAAqC;;QAErC,+CAA+C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwB/C,CAAC;AAEH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,sCAAsC;AACtC,QAAA,MAAM,iBAAiB;IACrB,6BAA6B;;IAE7B,iCAAiC;;IAEjC,4BAA4B;;IAE5B,mCAAmC;;QAhDnC,6BAA6B;;QAE7B,mCAAmC;;QAEnC,8EAA8E;;;;;;;;;;;IA8C9E,+DAA+D;;;;;;;;;;;;;;;;;;;;;;EAE/D,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAClD,wBAAwB;AACxB,eAAO,MAAM,YAAY;IAdvB,6BAA6B;;IAE7B,iCAAiC;;IAEjC,4BAA4B;;IAE5B,mCAAmC;;QAhDnC,6BAA6B;;QAE7B,mCAAmC;;QAEnC,8EAA8E;;;;;;;;;;;IA8C9E,+DAA+D;;;;;;;;;;;;;;;;;;;;;;WAMX,CAAC;AAEvD,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,2BAA2B;AAC3B,eAAO,MAAM,gBAAgB;IAC3B,yCAAyC;;IAEzC,+BAA+B;;QArB/B,6BAA6B;;QAE7B,iCAAiC;;QAEjC,4BAA4B;;QAE5B,mCAAmC;;YAhDnC,6BAA6B;;YAE7B,mCAAmC;;YAEnC,8EAA8E;;;;;;;;;;;QA8C9E,+DAA+D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe/D,CAAC;AAEH,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAChD,gDAAgD;AAChD,eAAO,MAAM,WAAW;IARtB,yCAAyC;;IAEzC,+BAA+B;;QArB/B,6BAA6B;;QAE7B,iCAAiC;;QAEjC,4BAA4B;;QAE5B,mCAAmC;;YAhDnC,6BAA6B;;YAE7B,mCAAmC;;YAEnC,8EAA8E;;;;;;;;;;;QA8C9E,+DAA+D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmBZ,CAAC;AAEtD,mDAAmD;AACnD,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAyBxB,CAAC"}
|
package/dist/types.js
CHANGED
package/dist/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAG3B,oBAAoB;AACpB,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,6BAA6B;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,mCAAmC;IACnC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;IAClD,8EAA8E;IAC9E,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACtE,CAAC,CAAC;AAGH,kCAAkC;AAClC,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,qCAAqC;IACrC,sBAAsB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC9C,+CAA+C;IAC/C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC;CAC7C,CAAC,CAAC;AAGH,8DAA8D;AAC9D,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,iCAAiC;IACjC,OAAO,EAAE,YAAY,CAAC,KAAK,EAAE;IAC7B,wDAAwD;IACxD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7B,sHAAsH;IACtH,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IACjC,kCAAkC;IAClC,KAAK,EAAE,wBAAwB,CAAC,QAAQ,EAAE;IAC1C,iCAAiC;IACjC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;CAC/C,CAAC,CAAC;AAGH,gEAAgE;AAChE,MAAM,CAAC,MAAM,yBAAyB,GAAG,mBAAmB,CAAC,MAAM,CAAC;IAClE,kCAAkC;IAClC,KAAK,EAAE,wBAAwB;CAChC,CAAC,CAAC;AAGH,sCAAsC;AACtC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,6BAA6B;IAC7B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,iCAAiC;IACjC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,4BAA4B;IAC5B,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE;IACrB,mCAAmC;IACnC,MAAM,EAAE,YAAY;IACpB,+DAA+D;IAC/D,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC/B,CAAC,CAAC;AAGH,wBAAwB;AACxB,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;AAGvD,2BAA2B;AAC3B,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,yCAAyC;IACzC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE;IAC1B,+BAA+B;IAC/B,IAAI,EAAE,iBAAiB;CACxB,CAAC,CAAC;AAGH,gDAAgD;AAChD,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;AAEtD,mDAAmD;AACnD,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE;IACnC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE;IACnC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACrC,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACxC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACnC,CAAC;KACD,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE;IACnB,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC;IAC5C,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACjD,CAAC;IACD,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,IAAI;QACJ,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,CAAC;KAC5C,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
package/dist/util.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAGA,wBAAsB,uBAAuB,CAAC,CAAC,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAGA,wBAAsB,uBAAuB,CAAC,CAAC,EAAE,CAAC,EAChD,KAAK,EAAE,CAAC,EAAE,EACV,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,GACnC,OAAO,CAAC,CAAC,EAAE,CAAC,CAGd;AAED,wBAAsB,QAAQ,CAAC,CAAC,EAAE,CAAC,EACjC,KAAK,EAAE,CAAC,EAAE,EACV,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GACvB,OAAO,CAAC,CAAC,EAAE,CAAC,CAOd;AAED,wBAAgB,eAAe,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,EAAE,GAAG,CAAC,EAAE,CAQhE"}
|
package/dist/util.js
CHANGED
|
@@ -9,6 +9,12 @@ export async function asyncMap(input, fn) {
|
|
|
9
9
|
return Promise.all(promises);
|
|
10
10
|
}
|
|
11
11
|
export function filterUndefined(input) {
|
|
12
|
-
|
|
12
|
+
const result = [];
|
|
13
|
+
for (const item of input) {
|
|
14
|
+
if (item !== undefined) {
|
|
15
|
+
result.push(item);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return result;
|
|
13
19
|
}
|
|
14
20
|
//# sourceMappingURL=util.js.map
|
package/dist/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,QAAQ,CAAC;AAE5B,8DAA8D;AAC9D,MAAM,CAAC,KAAK,UAAU,uBAAuB,
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,QAAQ,CAAC;AAE5B,8DAA8D;AAC9D,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,KAAU,EACV,EAAoC;IAEpC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC1C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAU,EACV,EAAwB;IAExB,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CACrB,KAAK,EACL,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAC1B,CAAC;IAEF,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,eAAe,CAAI,KAAwB;IACzD,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/package.json
CHANGED
|
@@ -6,28 +6,35 @@
|
|
|
6
6
|
"email": "npm@sjer.red",
|
|
7
7
|
"url": "https://sjer.red"
|
|
8
8
|
},
|
|
9
|
-
"homepage": "https://github.com/shepherdjerred/webring",
|
|
9
|
+
"homepage": "https://github.com/shepherdjerred/monorepo/tree/main/packages/webring",
|
|
10
10
|
"license": "GPL-3.0-only",
|
|
11
11
|
"repository": {
|
|
12
12
|
"type": "git",
|
|
13
|
-
"url": "https://github.com/shepherdjerred/
|
|
13
|
+
"url": "https://github.com/shepherdjerred/monorepo.git",
|
|
14
|
+
"directory": "packages/webring"
|
|
14
15
|
},
|
|
15
16
|
"bugs": {
|
|
16
|
-
"url": "https://github.com/shepherdjerred/
|
|
17
|
+
"url": "https://github.com/shepherdjerred/monorepo/issues"
|
|
17
18
|
},
|
|
18
19
|
"type": "module",
|
|
19
|
-
"version": "1.
|
|
20
|
+
"version": "1.6.0-dev.893",
|
|
20
21
|
"scripts": {
|
|
21
|
-
"prepare": "husky",
|
|
22
22
|
"lint": "eslint src --cache",
|
|
23
23
|
"build": "tsc",
|
|
24
24
|
"watch": "tsc -w",
|
|
25
|
-
"test": "
|
|
26
|
-
"test:update": "
|
|
27
|
-
"typedoc": "typedoc src/index.ts"
|
|
25
|
+
"test": "bun test src/",
|
|
26
|
+
"test:update": "bun test src/ --update-snapshots",
|
|
27
|
+
"typedoc": "typedoc src/index.ts",
|
|
28
|
+
"typecheck": "tsc --noEmit"
|
|
28
29
|
},
|
|
29
30
|
"main": "dist/index.js",
|
|
30
31
|
"types": "dist/index.d.ts",
|
|
32
|
+
"exports": {
|
|
33
|
+
".": {
|
|
34
|
+
"types": "./src/index.ts",
|
|
35
|
+
"default": "./dist/index.js"
|
|
36
|
+
}
|
|
37
|
+
},
|
|
31
38
|
"dependencies": {
|
|
32
39
|
"remeda": "^2.21.2",
|
|
33
40
|
"rss-parser": "^3.13.0",
|
|
@@ -36,39 +43,23 @@
|
|
|
36
43
|
"zod": "^3.24.2"
|
|
37
44
|
},
|
|
38
45
|
"devDependencies": {
|
|
39
|
-
"@
|
|
40
|
-
"@commitlint/config-conventional": "^20.0.0",
|
|
41
|
-
"@eslint/js": "^9.22.0",
|
|
46
|
+
"@shepherdjerred/eslint-config": "^0.2.0",
|
|
42
47
|
"@tsconfig/node24": "^24.0.0",
|
|
43
48
|
"@tsconfig/strictest": "^2.0.5",
|
|
49
|
+
"@types/bun": "^1.3.11",
|
|
44
50
|
"@types/express": "^5.0.0",
|
|
45
51
|
"@types/node": "^25.0.0",
|
|
46
52
|
"@types/sanitize-html": "^2.13.0",
|
|
47
53
|
"eslint": "^9.22.0",
|
|
48
54
|
"express": "^5.0.0",
|
|
49
55
|
"globals": "^17.0.0",
|
|
50
|
-
"
|
|
51
|
-
"lint-staged": "^16.0.0",
|
|
52
|
-
"prettier": "^3.5.3",
|
|
56
|
+
"jiti": "^2.6.1",
|
|
53
57
|
"typedoc": "^0.28.0",
|
|
54
58
|
"typedoc-plugin-extras": "^4.0.1",
|
|
55
59
|
"typedoc-plugin-zod": "^1.4.0",
|
|
56
|
-
"typescript": "^
|
|
57
|
-
"typescript-eslint": "^8.26.1",
|
|
60
|
+
"typescript": "^6",
|
|
58
61
|
"vitest": "^3.0.8"
|
|
59
62
|
},
|
|
60
|
-
"lint-staged": {
|
|
61
|
-
"*.{ts,tsx}": "eslint --cache --fix",
|
|
62
|
-
"*": "prettier --ignore-unknown --write"
|
|
63
|
-
},
|
|
64
|
-
"commitlint": {
|
|
65
|
-
"extends": [
|
|
66
|
-
"@commitlint/config-conventional"
|
|
67
|
-
]
|
|
68
|
-
},
|
|
69
|
-
"prettier": {
|
|
70
|
-
"printWidth": 120
|
|
71
|
-
},
|
|
72
63
|
"files": [
|
|
73
64
|
"dist",
|
|
74
65
|
"src",
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// Bun Snapshot v1, https://bun.sh/docs/test/snapshots
|
|
2
|
+
|
|
3
|
+
exports[`it should fetch an RSS feed without caching 1`] = `"[{"title":"Creating an already-completed asynchronous activity in C++/WinRT, part 4","url":"https://devblogs.microsoft.com/oldnewthing/20240712-00/?p=109967","date":"2024-07-12T14:00:00.000Z","source":{"title":"rss 10","url":"http://localhost:PORT/rss-10.xml"},"preview":"Failing is easy. Failing correctly is hard. The post Creating an already-completed asynchronous activity in C++/WinRT, part 4 appeared first on The Old New Thing."}]"`;
|
|
4
|
+
|
|
5
|
+
exports[`it should fetch several RSS feeds 1`] = `"[{"title":"Creating an already-completed asynchronous activity in C++/WinRT, part 4","url":"https://devblogs.microsoft.com/oldnewthing/20240712-00/?p=109967","date":"2024-07-12T14:00:00.000Z","source":{"title":"rss 10","url":"http://localhost:PORT/rss-10.xml"},"preview":"Failing is easy. Failing correctly is hard. The post Creating an already-completed asynchronous activity in C++/WinRT, part 4 appeared first on The Old New Thing."},{"title":"Garlic and gravel","url":"https://www.henrikkarlsson.xyz/p/garlic-and-gravel","date":"2024-07-12T11:34:25.000Z","source":{"title":"rss 16","url":"http://localhost:PORT/rss-16.xml"},"preview":"fragments"},{"title":"My Glorious Ascension To Thought Leadership","url":"https://ludic.mataroa.blog/blog/my-glorious-ascension-to-thought-leadership/","date":"2024-07-11T00:00:00.000Z","source":{"title":"rss 5","url":"http://localhost:PORT/rss-5.xml"},"preview":"I. This Escalated Quickly I mentioned one post ago that this blog has done approximately 2M+ lifetime views, with over a million of them being in the past two weeks in response to the now-infamous rant about AI, then I went out of my way to not write the most maximally viral thing possible for perso..."}]"`;
|
|
6
|
+
|
|
7
|
+
exports[`it should fetch an RSS feed with caching 1`] = `"[{"title":"Creating an already-completed asynchronous activity in C++/WinRT, part 4","url":"https://devblogs.microsoft.com/oldnewthing/20240712-00/?p=109967","date":"2024-07-12T14:00:00.000Z","source":{"url":"http://localhost:PORT/rss-10.xml","title":"rss 10"},"preview":"Failing is easy. Failing correctly is hard. The post Creating an already-completed asynchronous activity in C++/WinRT, part 4 appeared first on The Old New Thing."}]"`;
|
package/src/cache.ts
CHANGED
|
@@ -8,27 +8,34 @@ import {
|
|
|
8
8
|
type CachedConfiguration,
|
|
9
9
|
CacheSchema,
|
|
10
10
|
type CacheConfiguration,
|
|
11
|
-
} from "./types.
|
|
12
|
-
import { fetch } from "./fetch.
|
|
13
|
-
import { asyncMapFilterUndefined } from "./util.
|
|
14
|
-
import * as fs from "fs/promises";
|
|
11
|
+
} from "./types.ts";
|
|
12
|
+
import { fetch } from "./fetch.ts";
|
|
13
|
+
import { asyncMapFilterUndefined } from "./util.ts";
|
|
14
|
+
import * as fs from "node:fs/promises";
|
|
15
15
|
|
|
16
|
-
async function loadCache({
|
|
16
|
+
async function loadCache({
|
|
17
|
+
cache_file: cacheFilePath,
|
|
18
|
+
}: CacheConfiguration): Promise<Cache> {
|
|
17
19
|
try {
|
|
18
|
-
await fs.access(
|
|
19
|
-
const
|
|
20
|
-
return CacheSchema.parse(JSON.parse(
|
|
21
|
-
} catch
|
|
20
|
+
await fs.access(cacheFilePath);
|
|
21
|
+
const cacheFileContent = await fs.readFile(cacheFilePath);
|
|
22
|
+
return CacheSchema.parse(JSON.parse(cacheFileContent.toString()));
|
|
23
|
+
} catch {
|
|
22
24
|
return {};
|
|
23
25
|
}
|
|
24
26
|
}
|
|
25
27
|
|
|
26
|
-
async function saveCache(
|
|
27
|
-
|
|
28
|
+
async function saveCache(
|
|
29
|
+
{ cache_file: cacheFilePath }: CacheConfiguration,
|
|
30
|
+
cache: Cache,
|
|
31
|
+
) {
|
|
32
|
+
const dir = cacheFilePath.split("/").slice(0, -1).join("/");
|
|
28
33
|
if (dir !== "") {
|
|
29
|
-
await fs.mkdir(
|
|
34
|
+
await fs.mkdir(cacheFilePath.split("/").slice(0, -1).join("/"), {
|
|
35
|
+
recursive: true,
|
|
36
|
+
});
|
|
30
37
|
}
|
|
31
|
-
await fs.writeFile(
|
|
38
|
+
await fs.writeFile(cacheFilePath, JSON.stringify(cache));
|
|
32
39
|
}
|
|
33
40
|
|
|
34
41
|
function toCacheEntry(result: ResultEntry, now: Date): [string, CacheEntry] {
|
|
@@ -43,16 +50,23 @@ function toCache(results: ResultEntry[], now: Date): Cache {
|
|
|
43
50
|
);
|
|
44
51
|
}
|
|
45
52
|
|
|
46
|
-
function updateCache(
|
|
53
|
+
function updateCache(
|
|
54
|
+
results: ResultEntry[],
|
|
55
|
+
config: CachedConfiguration,
|
|
56
|
+
): Promise<void> {
|
|
47
57
|
const now = new Date();
|
|
48
58
|
const updatedCache = toCache(results, now);
|
|
49
59
|
return saveCache(config.cache, updatedCache);
|
|
50
60
|
}
|
|
51
61
|
|
|
52
|
-
export async function fetchAllCached(
|
|
62
|
+
export async function fetchAllCached(
|
|
63
|
+
config: CachedConfiguration,
|
|
64
|
+
): Promise<Result> {
|
|
53
65
|
const cache = await loadCache(config.cache);
|
|
54
66
|
|
|
55
|
-
const results = await asyncMapFilterUndefined(config.sources, (source) =>
|
|
67
|
+
const results = await asyncMapFilterUndefined(config.sources, (source) =>
|
|
68
|
+
fetchWithCache(source, cache, config),
|
|
69
|
+
);
|
|
56
70
|
|
|
57
71
|
await updateCache(results, config);
|
|
58
72
|
|
|
@@ -67,16 +81,19 @@ export async function fetchWithCache(
|
|
|
67
81
|
const cacheEntry = cache[source.url];
|
|
68
82
|
if (cacheEntry) {
|
|
69
83
|
const now = new Date();
|
|
70
|
-
if (
|
|
71
|
-
|
|
72
|
-
|
|
84
|
+
if (
|
|
85
|
+
now.getTime() - cacheEntry.timestamp.getTime() <
|
|
86
|
+
config.cache.cache_duration_minutes * 60 * 1000
|
|
87
|
+
) {
|
|
88
|
+
console.warn(`Cache entry found for ${source.url}.`);
|
|
89
|
+
return cacheEntry.data;
|
|
73
90
|
} else {
|
|
74
|
-
console.
|
|
91
|
+
console.warn(`Cache entry for ${source.url} is too old.`);
|
|
75
92
|
}
|
|
76
93
|
} else {
|
|
77
|
-
console.
|
|
94
|
+
console.warn(`No cache entry for ${source.url}.`);
|
|
78
95
|
}
|
|
79
96
|
|
|
80
|
-
console.
|
|
97
|
+
console.warn(`Fetching ${source.url}`);
|
|
81
98
|
return fetch(source, config.truncate);
|
|
82
99
|
}
|