webring 0.0.0-dev.706
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +280 -0
- package/README.md +153 -0
- package/bun.lock +1241 -0
- package/dist/cache.d.ts +4 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +61 -0
- package/dist/cache.js.map +1 -0
- package/dist/fetch.d.ts +14 -0
- package/dist/fetch.d.ts.map +1 -0
- package/dist/fetch.js +56 -0
- package/dist/fetch.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +33 -0
- package/dist/index.js.map +1 -0
- package/dist/index.test.d.ts +2 -0
- package/dist/index.test.d.ts.map +1 -0
- package/dist/index.test.js +69 -0
- package/dist/index.test.js.map +1 -0
- package/dist/types.d.ts +467 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +87 -0
- package/dist/types.js.map +1 -0
- package/dist/util.d.ts +4 -0
- package/dist/util.d.ts.map +1 -0
- package/dist/util.js +20 -0
- package/dist/util.js.map +1 -0
- package/package.json +72 -0
- package/src/__snapshots__/index.test.ts.snap +7 -0
- package/src/cache.ts +99 -0
- package/src/fetch.ts +84 -0
- package/src/index.test.ts +81 -0
- package/src/index.ts +50 -0
- package/src/types.ts +103 -0
- package/src/util.ts +32 -0
package/dist/cache.d.ts
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type Cache, type Result, type ResultEntry, type Source, type CachedConfiguration } from "./types.ts";
|
|
2
|
+
export declare function fetchAllCached(config: CachedConfiguration): Promise<Result>;
|
|
3
|
+
export declare function fetchWithCache(source: Source, cache: Cache, config: CachedConfiguration): Promise<ResultEntry | undefined>;
|
|
4
|
+
//# sourceMappingURL=cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AACA,OAAO,EACL,KAAK,KAAK,EACV,KAAK,MAAM,EACX,KAAK,WAAW,EAEhB,KAAK,MAAM,EACX,KAAK,mBAAmB,EAGzB,MAAM,YAAY,CAAC;AAmDpB,wBAAsB,cAAc,CAClC,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,MAAM,CAAC,CAUjB;AAED,wBAAsB,cAAc,CAClC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAmBlC"}
|
package/dist/cache.js
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import * as R from "remeda";
|
|
2
|
+
import { CacheSchema, } from "./types.js";
|
|
3
|
+
import { fetch } from "./fetch.js";
|
|
4
|
+
import { asyncMapFilterUndefined } from "./util.js";
|
|
5
|
+
import * as fs from "node:fs/promises";
|
|
6
|
+
async function loadCache({ cache_file: cacheFilePath, }) {
|
|
7
|
+
try {
|
|
8
|
+
await fs.access(cacheFilePath);
|
|
9
|
+
const cacheFileContent = await fs.readFile(cacheFilePath);
|
|
10
|
+
return CacheSchema.parse(JSON.parse(cacheFileContent.toString()));
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return {};
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
async function saveCache({ cache_file: cacheFilePath }, cache) {
|
|
17
|
+
const dir = cacheFilePath.split("/").slice(0, -1).join("/");
|
|
18
|
+
if (dir !== "") {
|
|
19
|
+
await fs.mkdir(cacheFilePath.split("/").slice(0, -1).join("/"), {
|
|
20
|
+
recursive: true,
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
await fs.writeFile(cacheFilePath, JSON.stringify(cache));
|
|
24
|
+
}
|
|
25
|
+
function toCacheEntry(result, now) {
|
|
26
|
+
return [result.source.url, { timestamp: now, data: result }];
|
|
27
|
+
}
|
|
28
|
+
function toCache(results, now) {
|
|
29
|
+
return R.pipe(results, R.map((result) => toCacheEntry(result, now)), R.fromEntries());
|
|
30
|
+
}
|
|
31
|
+
function updateCache(results, config) {
|
|
32
|
+
const now = new Date();
|
|
33
|
+
const updatedCache = toCache(results, now);
|
|
34
|
+
return saveCache(config.cache, updatedCache);
|
|
35
|
+
}
|
|
36
|
+
export async function fetchAllCached(config) {
|
|
37
|
+
const cache = await loadCache(config.cache);
|
|
38
|
+
const results = await asyncMapFilterUndefined(config.sources, (source) => fetchWithCache(source, cache, config));
|
|
39
|
+
await updateCache(results, config);
|
|
40
|
+
return results;
|
|
41
|
+
}
|
|
42
|
+
export async function fetchWithCache(source, cache, config) {
|
|
43
|
+
const cacheEntry = cache[source.url];
|
|
44
|
+
if (cacheEntry) {
|
|
45
|
+
const now = new Date();
|
|
46
|
+
if (now.getTime() - cacheEntry.timestamp.getTime() <
|
|
47
|
+
config.cache.cache_duration_minutes * 60 * 1000) {
|
|
48
|
+
console.warn(`Cache entry found for ${source.url}.`);
|
|
49
|
+
return cacheEntry.data;
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
console.warn(`Cache entry for ${source.url} is too old.`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
console.warn(`No cache entry for ${source.url}.`);
|
|
57
|
+
}
|
|
58
|
+
console.warn(`Fetching ${source.url}`);
|
|
59
|
+
return fetch(source, config.truncate);
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAOL,WAAW,GAEZ,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEvC,KAAK,UAAU,SAAS,CAAC,EACvB,UAAU,EAAE,aAAa,GACN;IACnB,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAC/B,MAAM,gBAAgB,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC1D,OAAO,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,EAAE,UAAU,EAAE,aAAa,EAAsB,EACjD,KAAY;IAEZ,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5D,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YAC9D,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;IACL,CAAC;IACD,MAAM,EAAE,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,YAAY,CAAC,MAAmB,EAAE,GAAS;IAClD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,OAAO,CAAC,OAAsB,EAAE,GAAS;IAChD,OAAO,CAAC,CAAC,IAAI,CACX,OAAO,EACP,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,EAC5C,CAAC,CAAC,WAAW,EAAE,CAChB,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,OAAsB,EACtB,MAA2B;IAE3B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC3C,OAAO,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAA2B;IAE3B,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAE5C,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CACvE,cAAc,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CACtC,CAAC;IAEF,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAEnC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,KAAY,EACZ,MAA2B;IAE3B,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IACE,GAAG,CAAC,OAAO,EAAE,GAAG,UAAU,CAAC,SAAS,CAAC,OAAO,EAAE;YAC9C,MAAM,CAAC,KAAK,CAAC,sBAAsB,GAAG,EAAE,GAAG,IAAI,EAC/C,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;YACrD,OAAO,UAAU,CAAC,IAAI,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,sBAAsB,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IACvC,OAAO,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;AACxC,CAAC"}
|
package/dist/fetch.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type Source, type ResultEntry, type Configuration } from "./types.ts";
|
|
2
|
+
export declare function fetchAll(config: Configuration): Promise<{
|
|
3
|
+
url: string;
|
|
4
|
+
title: string;
|
|
5
|
+
date: Date;
|
|
6
|
+
source: {
|
|
7
|
+
url: string;
|
|
8
|
+
title: string;
|
|
9
|
+
filter?: ((args_0: string, ...args: unknown[]) => boolean) | undefined;
|
|
10
|
+
};
|
|
11
|
+
preview?: string | undefined;
|
|
12
|
+
}[]>;
|
|
13
|
+
export declare function fetch(source: Source, length: number): Promise<ResultEntry | undefined>;
|
|
14
|
+
//# sourceMappingURL=fetch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,MAAM,EACX,KAAK,WAAW,EAEhB,KAAK,aAAa,EACnB,MAAM,YAAY,CAAC;AAwBpB,wBAAsB,QAAQ,CAAC,MAAM,EAAE,aAAa;;;;;;;;;;KAInD;AAED,wBAAsB,KAAK,CACzB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAyClC"}
|
package/dist/fetch.js
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import Parser from "rss-parser";
|
|
2
|
+
import sanitizeHtml from "sanitize-html";
|
|
3
|
+
import * as truncateHtml from "truncate-html";
|
|
4
|
+
import { z } from "zod/v3";
|
|
5
|
+
import { FeedEntrySchema, } from "./types.js";
|
|
6
|
+
import * as R from "remeda";
|
|
7
|
+
import { asyncMapFilterUndefined } from "./util.js";
|
|
8
|
+
// Handle ESM/CommonJS interop - truncate-html exports differently in different environments
|
|
9
|
+
const TruncateFnSchema = z
|
|
10
|
+
.function()
|
|
11
|
+
.args(z.string(), z.number().optional())
|
|
12
|
+
.returns(z.string());
|
|
13
|
+
const TruncateModuleSchema = z.object({ default: TruncateFnSchema });
|
|
14
|
+
function truncate(html, length) {
|
|
15
|
+
const mod = truncateHtml;
|
|
16
|
+
const fnResult = TruncateFnSchema.safeParse(mod);
|
|
17
|
+
if (fnResult.success) {
|
|
18
|
+
return fnResult.data(html, length);
|
|
19
|
+
}
|
|
20
|
+
const modResult = TruncateModuleSchema.safeParse(mod);
|
|
21
|
+
if (modResult.success) {
|
|
22
|
+
return modResult.data.default(html, length);
|
|
23
|
+
}
|
|
24
|
+
throw new Error("truncate-html module could not be resolved");
|
|
25
|
+
}
|
|
26
|
+
export async function fetchAll(config) {
|
|
27
|
+
return await asyncMapFilterUndefined(config.sources, (source) => fetch(source, config.truncate));
|
|
28
|
+
}
|
|
29
|
+
export async function fetch(source, length) {
|
|
30
|
+
const parser = new Parser();
|
|
31
|
+
try {
|
|
32
|
+
const feed = await parser.parseURL(source.url);
|
|
33
|
+
const firstItem = R.pipe(feed.items, R.map((item) => FeedEntrySchema.parse(item)), R.sortBy((item) => new Date(item.date).getTime()), R.reverse(), R.first());
|
|
34
|
+
if (!firstItem) {
|
|
35
|
+
throw new Error("no items found in feed");
|
|
36
|
+
}
|
|
37
|
+
const preview = firstItem.contentSnippet ??
|
|
38
|
+
firstItem.content ??
|
|
39
|
+
firstItem.description ??
|
|
40
|
+
firstItem["content:encoded"];
|
|
41
|
+
return {
|
|
42
|
+
title: firstItem.title,
|
|
43
|
+
url: firstItem.link,
|
|
44
|
+
date: new Date(firstItem.date),
|
|
45
|
+
source,
|
|
46
|
+
preview: preview !== undefined && preview !== ""
|
|
47
|
+
? truncate(sanitizeHtml(preview, { parseStyleAttributes: false }), length)
|
|
48
|
+
: undefined,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
console.error(`Error fetching ${source.url}: ${String(error)}`);
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=fetch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,YAAY,CAAC;AAChC,OAAO,YAAY,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,YAAY,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAC3B,OAAO,EAGL,eAAe,GAEhB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,CAAC,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAC;AAEpD,4FAA4F;AAC5F,MAAM,gBAAgB,GAAG,CAAC;KACvB,QAAQ,EAAE;KACV,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC;KACvC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;AACvB,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;AAErE,SAAS,QAAQ,CAAC,IAAY,EAAE,MAAe;IAC7C,MAAM,GAAG,GAAG,YAAuB,CAAC;IACpC,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjD,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,SAAS,GAAG,oBAAoB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACtD,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAqB;IAClD,OAAO,MAAM,uBAAuB,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAC9D,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,CAC/B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,MAAc,EACd,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAE5B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CACtB,IAAI,CAAC,KAAK,EACV,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAC5C,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,EACjD,CAAC,CAAC,OAAO,EAAE,EACX,CAAC,CAAC,KAAK,EAAE,CACV,CAAC;QAEF,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,OAAO,GACX,SAAS,CAAC,cAAc;YACxB,SAAS,CAAC,OAAO;YACjB,SAAS,CAAC,WAAW;YACrB,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAE/B,OAAO;YACL,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,GAAG,EAAE,SAAS,CAAC,IAAI;YACnB,IAAI,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC9B,MAAM;YACN,OAAO,EACL,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE;gBACrC,CAAC,CAAC,QAAQ,CACN,YAAY,CAAC,OAAO,EAAE,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC,EACtD,MAAM,CACP;gBACH,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,kBAAkB,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChE,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,MAAM,EAEZ,MAAM,YAAY,CAAC;AAEpB,YAAY,EAAE,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAErE,wBAAsB,GAAG,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAsChE"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import * as R from "remeda";
|
|
2
|
+
import { fetchAllCached } from "./cache.js";
|
|
3
|
+
import { fetchAll as fetchAllUncached } from "./fetch.js";
|
|
4
|
+
import { CachedConfigurationSchema, } from "./types.js";
|
|
5
|
+
export async function run(config) {
|
|
6
|
+
const { success, data } = CachedConfigurationSchema.safeParse(config);
|
|
7
|
+
let fetched;
|
|
8
|
+
if (success) {
|
|
9
|
+
console.warn(`Using cache at ${data.cache.cache_file}.`);
|
|
10
|
+
fetched = await fetchAllCached(data);
|
|
11
|
+
}
|
|
12
|
+
else {
|
|
13
|
+
console.warn("Cache disabled.");
|
|
14
|
+
fetched = await fetchAllUncached(config);
|
|
15
|
+
}
|
|
16
|
+
let results = R.pipe(fetched, R.sortBy((entry) => entry.date.getTime()), R.reverse(), R.filter((entry) => {
|
|
17
|
+
const filterFn = entry.source.filter;
|
|
18
|
+
if (filterFn === undefined ||
|
|
19
|
+
entry.preview === undefined ||
|
|
20
|
+
entry.preview === "") {
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
return filterFn(entry.preview);
|
|
24
|
+
}));
|
|
25
|
+
// shuffle if wanted
|
|
26
|
+
if (config.shuffle === true) {
|
|
27
|
+
results = R.shuffle(results);
|
|
28
|
+
}
|
|
29
|
+
// take n
|
|
30
|
+
results = R.take(results, config.number);
|
|
31
|
+
return results;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,QAAQ,IAAI,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAGL,yBAAyB,GAC1B,MAAM,YAAY,CAAC;AAIpB,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,MAAqB;IAC7C,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,yBAAyB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAEtE,IAAI,OAAe,CAAC;IACpB,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC;QACzD,OAAO,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAChC,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,IAAI,CAClB,OAAO,EACP,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EACzC,CAAC,CAAC,OAAO,EAAE,EACX,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QACjB,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;QACrC,IACE,QAAQ,KAAK,SAAS;YACtB,KAAK,CAAC,OAAO,KAAK,SAAS;YAC3B,KAAK,CAAC,OAAO,KAAK,EAAE,EACpB,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC,CAAC,CACH,CAAC;IAEF,oBAAoB;IACpB,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QAC5B,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAED,SAAS;IACT,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAEzC,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.test.d.ts","sourceRoot":"","sources":["../src/index.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { expect, test } from "bun:test";
|
|
2
|
+
import { run } from "./index.js";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { mkdtemp } from "node:fs/promises";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import express from "express";
|
|
7
|
+
const app = express();
|
|
8
|
+
app.use(express.static("src/testdata"));
|
|
9
|
+
const port = Math.floor(Math.random() * 10_000) + 3000;
|
|
10
|
+
app.listen(port, () => {
|
|
11
|
+
console.warn(`Test server listening at http://localhost:${port.toString()}`);
|
|
12
|
+
});
|
|
13
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
14
|
+
function createUrl(urlPath) {
|
|
15
|
+
return `http://localhost:${port.toString()}/${urlPath}`;
|
|
16
|
+
}
|
|
17
|
+
function createSources(count) {
|
|
18
|
+
return Array.from({ length: count }, (_, i) => ({
|
|
19
|
+
title: `rss ${i.toString()}`,
|
|
20
|
+
url: createUrl(`rss-${i.toString()}.xml`),
|
|
21
|
+
}));
|
|
22
|
+
}
|
|
23
|
+
test("it should fetch an RSS feed without caching", async () => {
|
|
24
|
+
const config = {
|
|
25
|
+
sources: createSources(19),
|
|
26
|
+
number: 1,
|
|
27
|
+
truncate: 300,
|
|
28
|
+
};
|
|
29
|
+
const result = await run(config);
|
|
30
|
+
let string = JSON.stringify(result);
|
|
31
|
+
// replace the port number with a fixed value
|
|
32
|
+
string = string.replaceAll(port.toString(), "PORT");
|
|
33
|
+
expect(string).toMatchSnapshot();
|
|
34
|
+
});
|
|
35
|
+
test("it should fetch several RSS feeds", async () => {
|
|
36
|
+
const config = {
|
|
37
|
+
sources: createSources(19),
|
|
38
|
+
number: 3,
|
|
39
|
+
truncate: 300,
|
|
40
|
+
};
|
|
41
|
+
const result = await run(config);
|
|
42
|
+
let string = JSON.stringify(result);
|
|
43
|
+
// replace the port number with a fixed value
|
|
44
|
+
string = string.replaceAll(port.toString(), "PORT");
|
|
45
|
+
expect(string).toMatchSnapshot();
|
|
46
|
+
});
|
|
47
|
+
test("it should fetch an RSS feed with caching", async () => {
|
|
48
|
+
const config = {
|
|
49
|
+
sources: createSources(19),
|
|
50
|
+
number: 1,
|
|
51
|
+
truncate: 300,
|
|
52
|
+
cache: {
|
|
53
|
+
cache_file: `${await createTempDir()}/cache.json`,
|
|
54
|
+
cache_duration_minutes: 1,
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
const result = await run(config);
|
|
58
|
+
let string = JSON.stringify(result);
|
|
59
|
+
// replace the port number with a fixed value
|
|
60
|
+
string = string.replaceAll(port.toString(), "PORT");
|
|
61
|
+
expect(string).toMatchSnapshot();
|
|
62
|
+
});
|
|
63
|
+
// https://sdorra.dev/posts/2024-02-12-vitest-tmpdir
|
|
64
|
+
async function createTempDir() {
|
|
65
|
+
const ostmpdir = tmpdir();
|
|
66
|
+
const dir = path.join(ostmpdir, "unit-test-");
|
|
67
|
+
return await mkdtemp(dir);
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=index.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../src/index.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAExC,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,MAAM,GAAG,GAAG,OAAO,EAAE,CAAC;AACtB,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;AAExC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;AACvD,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;IACpB,OAAO,CAAC,IAAI,CAAC,6CAA6C,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC,CAAC,CAAC;AAEH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;AAEzD,SAAS,SAAS,CAAC,OAAe;IAChC,OAAO,oBAAoB,IAAI,CAAC,QAAQ,EAAE,IAAI,OAAO,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9C,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC5B,GAAG,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC;KAC1C,CAAC,CAAC,CAAC;AACN,CAAC;AAED,IAAI,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;IAC7D,MAAM,MAAM,GAAkB;QAC5B,OAAO,EAAE,aAAa,CAAC,EAAE,CAAC;QAC1B,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,GAAG;KACd,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpC,6CAA6C;IAC7C,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,EAAE,CAAC;AACnC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;IACnD,MAAM,MAAM,GAAkB;QAC5B,OAAO,EAAE,aAAa,CAAC,EAAE,CAAC;QAC1B,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,GAAG;KACd,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpC,6CAA6C;IAC7C,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,EAAE,CAAC;AACnC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;IAC1D,MAAM,MAAM,GAAkB;QAC5B,OAAO,EAAE,aAAa,CAAC,EAAE,CAAC;QAC1B,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,GAAG;QACb,KAAK,EAAE;YACL,UAAU,EAAE,GAAG,MAAM,aAAa,EAAE,aAAa;YACjD,sBAAsB,EAAE,CAAC;SAC1B;KACF,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpC,6CAA6C;IAC7C,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,EAAE,CAAC;AACnC,CAAC,CAAC,CAAC;AAEH,oDAAoD;AACpD,KAAK,UAAU,aAAa;IAC1B,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC;IAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC9C,OAAO,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;AAC5B,CAAC"}
|