webring 1.7.1-dev.5538 → 1.7.1-dev.5768
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fetch.d.ts.map +1 -1
- package/dist/fetch.js +81 -18
- package/dist/fetch.js.map +1 -1
- package/package.json +7 -8
- package/src/fetch.ts +87 -18
- package/bun.lock +0 -1234
package/dist/fetch.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,MAAM,EACX,KAAK,WAAW,EAEhB,KAAK,aAAa,EACnB,MAAM,YAAY,CAAC;AA8FpB,wBAAsB,QAAQ,CAAC,MAAM,EAAE,aAAa;;;;;;;;;;KAInD;AAED,wBAAsB,KAAK,CACzB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAyClC"}
|
package/dist/fetch.js
CHANGED
|
@@ -1,27 +1,90 @@
|
|
|
1
1
|
import Parser from "rss-parser";
|
|
2
2
|
import sanitizeHtml from "sanitize-html";
|
|
3
|
-
import
|
|
4
|
-
import { z } from "zod/v3";
|
|
3
|
+
import { Parser as HtmlParser } from "htmlparser2";
|
|
5
4
|
import { FeedEntrySchema, } from "./types.js";
|
|
6
5
|
import * as R from "remeda";
|
|
7
6
|
import { asyncMapFilterUndefined } from "./util.js";
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
7
|
+
const VOID_TAGS = new Set([
|
|
8
|
+
"area",
|
|
9
|
+
"base",
|
|
10
|
+
"br",
|
|
11
|
+
"col",
|
|
12
|
+
"embed",
|
|
13
|
+
"hr",
|
|
14
|
+
"img",
|
|
15
|
+
"input",
|
|
16
|
+
"link",
|
|
17
|
+
"meta",
|
|
18
|
+
"source",
|
|
19
|
+
"track",
|
|
20
|
+
"wbr",
|
|
21
|
+
]);
|
|
22
|
+
function escapeText(text) {
|
|
23
|
+
return text
|
|
24
|
+
.replaceAll("&", "&")
|
|
25
|
+
.replaceAll("<", "<")
|
|
26
|
+
.replaceAll(">", ">");
|
|
27
|
+
}
|
|
28
|
+
function escapeAttributeValue(value) {
|
|
29
|
+
return escapeText(value).replaceAll('"', """);
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Truncate HTML to `length` text characters while keeping tags balanced.
|
|
33
|
+
* Whitespace runs collapse to a single space and `...` marks a cut, matching
|
|
34
|
+
* the behavior of the retired `truncate-html` package (which was pinned to a
|
|
35
|
+
* dead cheerio 1.0.0-rc.12/htmlparser2@6 chain).
|
|
36
|
+
*/
|
|
37
|
+
function truncate(html, length = 300) {
|
|
38
|
+
let out = "";
|
|
39
|
+
let remaining = length;
|
|
40
|
+
let truncated = false;
|
|
41
|
+
const openTags = [];
|
|
42
|
+
const parser = new HtmlParser({
|
|
43
|
+
onopentag(name, attributes) {
|
|
44
|
+
if (truncated) {
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const attributeText = Object.entries(attributes)
|
|
48
|
+
.map(([key, value]) => value === "" ? ` ${key}` : ` ${key}="${escapeAttributeValue(value)}"`)
|
|
49
|
+
.join("");
|
|
50
|
+
out += `<${name}${attributeText}>`;
|
|
51
|
+
if (!VOID_TAGS.has(name)) {
|
|
52
|
+
openTags.push(name);
|
|
53
|
+
}
|
|
54
|
+
},
|
|
55
|
+
ontext(text) {
|
|
56
|
+
if (truncated) {
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
const collapsed = text.replaceAll(/\s+/g, " ");
|
|
60
|
+
if (collapsed.length <= remaining) {
|
|
61
|
+
out += escapeText(collapsed);
|
|
62
|
+
remaining -= collapsed.length;
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
out += `${escapeText(collapsed.slice(0, remaining))}...`;
|
|
66
|
+
truncated = true;
|
|
67
|
+
},
|
|
68
|
+
onclosetag(name) {
|
|
69
|
+
if (VOID_TAGS.has(name)) {
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
// Only emit closes for tags we actually opened (tags opened after the
|
|
73
|
+
// cut never enter the stack, so their close events are dropped too).
|
|
74
|
+
const index = openTags.lastIndexOf(name);
|
|
75
|
+
if (index === -1) {
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
openTags.splice(index, 1);
|
|
79
|
+
out += `</${name}>`;
|
|
80
|
+
},
|
|
81
|
+
});
|
|
82
|
+
parser.write(html);
|
|
83
|
+
parser.end();
|
|
84
|
+
for (const name of openTags.reverse()) {
|
|
85
|
+
out += `</${name}>`;
|
|
23
86
|
}
|
|
24
|
-
|
|
87
|
+
return out;
|
|
25
88
|
}
|
|
26
89
|
export async function fetchAll(config) {
|
|
27
90
|
return await asyncMapFilterUndefined(config.sources, (source) => fetch(source, config.truncate));
|
package/dist/fetch.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,YAAY,CAAC;AAChC,OAAO,YAAY,MAAM,eAAe,CAAC;AACzC,OAAO,
|
|
1
|
+
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,YAAY,CAAC;AAChC,OAAO,YAAY,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAGL,eAAe,GAEhB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,CAAC,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAC;AAEpD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,MAAM;IACN,MAAM;IACN,IAAI;IACJ,KAAK;IACL,OAAO;IACP,IAAI;IACJ,KAAK;IACL,OAAO;IACP,MAAM;IACN,MAAM;IACN,QAAQ;IACR,OAAO;IACP,KAAK;CACN,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,IAAI;SACR,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC;SACxB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC;SACvB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IACzC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AACrD,CAAC;AAED;;;;;GAKG;AACH,SAAS,QAAQ,CAAC,IAAY,EAAE,MAAM,GAAG,GAAG;IAC1C,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,SAAS,GAAG,MAAM,CAAC;IACvB,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC;QAC5B,SAAS,CAAC,IAAI,EAAE,UAAU;YACxB,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO;YACT,CAAC;YACD,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC;iBAC7C,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACpB,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,oBAAoB,CAAC,KAAK,CAAC,GAAG,CACtE;iBACA,IAAI,CAAC,EAAE,CAAC,CAAC;YACZ,GAAG,IAAI,IAAI,IAAI,GAAG,aAAa,GAAG,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI;YACT,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO;YACT,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAC/C,IAAI,SAAS,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;gBAClC,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;gBAC7B,SAAS,IAAI,SAAS,CAAC,MAAM,CAAC;gBAC9B,OAAO;YACT,CAAC;YACD,GAAG,IAAI,GAAG,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC;YACzD,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC;QACD,UAAU,CAAC,IAAI;YACb,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,OAAO;YACT,CAAC;YACD,sEAAsE;YACtE,qEAAqE;YACrE,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;gBACjB,OAAO;YACT,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC1B,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;QACtB,CAAC;KACF,CAAC,CAAC;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnB,MAAM,CAAC,GAAG,EAAE,CAAC;IAEb,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACtC,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;IACtB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAqB;IAClD,OAAO,MAAM,uBAAuB,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAC9D,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,CAC/B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,MAAc,EACd,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAE5B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CACtB,IAAI,CAAC,KAAK,EACV,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAC5C,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,EACjD,CAAC,CAAC,OAAO,EAAE,EACX,CAAC,CAAC,KAAK,EAAE,CACV,CAAC;QAEF,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,OAAO,GACX,SAAS,CAAC,cAAc;YACxB,SAAS,CAAC,OAAO;YACjB,SAAS,CAAC,WAAW;YACrB,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAE/B,OAAO;YACL,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,GAAG,EAAE,SAAS,CAAC,IAAI;YACnB,IAAI,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC9B,MAAM;YACN,OAAO,EACL,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE;gBACrC,CAAC,CAAC,QAAQ,CACN,YAAY,CAAC,OAAO,EAAE,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC,EACtD,MAAM,CACP;gBACH,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,kBAAkB,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChE,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
"url": "https://github.com/shepherdjerred/monorepo/issues"
|
|
18
18
|
},
|
|
19
19
|
"type": "module",
|
|
20
|
-
"version": "1.7.1-dev.
|
|
20
|
+
"version": "1.7.1-dev.5768",
|
|
21
21
|
"scripts": {
|
|
22
22
|
"lint": "eslint src --cache",
|
|
23
23
|
"build": "tsc",
|
|
@@ -25,7 +25,9 @@
|
|
|
25
25
|
"test": "bun test src/",
|
|
26
26
|
"test:update": "bun test src/ --update-snapshots",
|
|
27
27
|
"typedoc": "typedoc src/index.ts",
|
|
28
|
-
"typecheck": "tsc --noEmit"
|
|
28
|
+
"typecheck": "tsc --noEmit",
|
|
29
|
+
"deploy": "bun ../../scripts/deploy-site.ts webring",
|
|
30
|
+
"publish:npm": "bun ../../scripts/publish-npm.ts ."
|
|
29
31
|
},
|
|
30
32
|
"main": "dist/index.js",
|
|
31
33
|
"types": "dist/index.d.ts",
|
|
@@ -36,14 +38,14 @@
|
|
|
36
38
|
}
|
|
37
39
|
},
|
|
38
40
|
"dependencies": {
|
|
41
|
+
"htmlparser2": "^10.0.0",
|
|
39
42
|
"remeda": "^2.33.7",
|
|
40
43
|
"rss-parser": "^3.13.0",
|
|
41
44
|
"sanitize-html": "^2.17.4",
|
|
42
|
-
"truncate-html": "^1.2.2",
|
|
43
45
|
"zod": "^4.4.3"
|
|
44
46
|
},
|
|
45
47
|
"devDependencies": {
|
|
46
|
-
"@shepherdjerred/eslint-config": "
|
|
48
|
+
"@shepherdjerred/eslint-config": "0.3.0",
|
|
47
49
|
"@tsconfig/node24": "^24.0.4",
|
|
48
50
|
"@tsconfig/strictest": "^2.0.8",
|
|
49
51
|
"@types/bun": "^1.3.13",
|
|
@@ -68,8 +70,5 @@
|
|
|
68
70
|
"LICENSE",
|
|
69
71
|
"bun.lock",
|
|
70
72
|
"CHANGELOG.md"
|
|
71
|
-
]
|
|
72
|
-
"overrides": {
|
|
73
|
-
"sanitize-html": "^2.17.4"
|
|
74
|
-
}
|
|
73
|
+
]
|
|
75
74
|
}
|
package/src/fetch.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import Parser from "rss-parser";
|
|
2
2
|
import sanitizeHtml from "sanitize-html";
|
|
3
|
-
import
|
|
4
|
-
import { z } from "zod/v3";
|
|
3
|
+
import { Parser as HtmlParser } from "htmlparser2";
|
|
5
4
|
import {
|
|
6
5
|
type Source,
|
|
7
6
|
type ResultEntry,
|
|
@@ -11,24 +10,94 @@ import {
|
|
|
11
10
|
import * as R from "remeda";
|
|
12
11
|
import { asyncMapFilterUndefined } from "./util.ts";
|
|
13
12
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
13
|
+
const VOID_TAGS = new Set([
|
|
14
|
+
"area",
|
|
15
|
+
"base",
|
|
16
|
+
"br",
|
|
17
|
+
"col",
|
|
18
|
+
"embed",
|
|
19
|
+
"hr",
|
|
20
|
+
"img",
|
|
21
|
+
"input",
|
|
22
|
+
"link",
|
|
23
|
+
"meta",
|
|
24
|
+
"source",
|
|
25
|
+
"track",
|
|
26
|
+
"wbr",
|
|
27
|
+
]);
|
|
20
28
|
|
|
21
|
-
function
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
29
|
+
function escapeText(text: string): string {
|
|
30
|
+
return text
|
|
31
|
+
.replaceAll("&", "&")
|
|
32
|
+
.replaceAll("<", "<")
|
|
33
|
+
.replaceAll(">", ">");
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function escapeAttributeValue(value: string): string {
|
|
37
|
+
return escapeText(value).replaceAll('"', """);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Truncate HTML to `length` text characters while keeping tags balanced.
|
|
42
|
+
* Whitespace runs collapse to a single space and `...` marks a cut, matching
|
|
43
|
+
* the behavior of the retired `truncate-html` package (which was pinned to a
|
|
44
|
+
* dead cheerio 1.0.0-rc.12/htmlparser2@6 chain).
|
|
45
|
+
*/
|
|
46
|
+
function truncate(html: string, length = 300): string {
|
|
47
|
+
let out = "";
|
|
48
|
+
let remaining = length;
|
|
49
|
+
let truncated = false;
|
|
50
|
+
const openTags: string[] = [];
|
|
51
|
+
|
|
52
|
+
const parser = new HtmlParser({
|
|
53
|
+
onopentag(name, attributes) {
|
|
54
|
+
if (truncated) {
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
const attributeText = Object.entries(attributes)
|
|
58
|
+
.map(([key, value]) =>
|
|
59
|
+
value === "" ? ` ${key}` : ` ${key}="${escapeAttributeValue(value)}"`,
|
|
60
|
+
)
|
|
61
|
+
.join("");
|
|
62
|
+
out += `<${name}${attributeText}>`;
|
|
63
|
+
if (!VOID_TAGS.has(name)) {
|
|
64
|
+
openTags.push(name);
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
ontext(text) {
|
|
68
|
+
if (truncated) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
const collapsed = text.replaceAll(/\s+/g, " ");
|
|
72
|
+
if (collapsed.length <= remaining) {
|
|
73
|
+
out += escapeText(collapsed);
|
|
74
|
+
remaining -= collapsed.length;
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
out += `${escapeText(collapsed.slice(0, remaining))}...`;
|
|
78
|
+
truncated = true;
|
|
79
|
+
},
|
|
80
|
+
onclosetag(name) {
|
|
81
|
+
if (VOID_TAGS.has(name)) {
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
// Only emit closes for tags we actually opened (tags opened after the
|
|
85
|
+
// cut never enter the stack, so their close events are dropped too).
|
|
86
|
+
const index = openTags.lastIndexOf(name);
|
|
87
|
+
if (index === -1) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
openTags.splice(index, 1);
|
|
91
|
+
out += `</${name}>`;
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
parser.write(html);
|
|
95
|
+
parser.end();
|
|
96
|
+
|
|
97
|
+
for (const name of openTags.reverse()) {
|
|
98
|
+
out += `</${name}>`;
|
|
30
99
|
}
|
|
31
|
-
|
|
100
|
+
return out;
|
|
32
101
|
}
|
|
33
102
|
|
|
34
103
|
export async function fetchAll(config: Configuration) {
|