webforai 1.5.1 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +258 -278
- package/dist/cjs/extract-hast/index.js +4 -3
- package/dist/cjs/extract-hast/readability.js +28 -61
- package/dist/cjs/extract-hast/strongly.js +2 -1
- package/dist/cjs/extract-hast/utils.js +0 -15
- package/dist/cjs/html-to-mdast.js +6 -6
- package/dist/cjs/link-replacer.js +10 -5
- package/dist/cjs/loaders/playwright.js +36 -7
- package/dist/cjs/{cli/constants.js → mdast-handlers/custom-br-handler.js} +9 -12
- package/dist/cjs/mdast-to-markdown.js +1 -1
- package/dist/cjs/utils/hast-utils.js +71 -0
- package/dist/extract-hast/index.js +4 -3
- package/dist/extract-hast/readability.js +29 -62
- package/dist/extract-hast/strongly.js +2 -1
- package/dist/extract-hast/utils.js +0 -14
- package/dist/html-to-mdast.js +6 -6
- package/dist/link-replacer.js +10 -5
- package/dist/loaders/playwright.js +36 -7
- package/dist/mdast-handlers/custom-br-handler.js +8 -0
- package/dist/mdast-to-markdown.js +1 -1
- package/dist/types/cli/bin.js.map +1 -1
- package/dist/types/cli/commands/webforai/index.d.ts +6 -0
- package/dist/types/cli/commands/webforai/index.js.map +1 -0
- package/dist/types/cli/commands/webforai/loadHtml.d.ts +3 -0
- package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -0
- package/dist/types/extract-hast/index.d.ts +9 -4
- package/dist/types/extract-hast/index.js.map +1 -1
- package/dist/types/extract-hast/readability.d.ts +2 -1
- package/dist/types/extract-hast/readability.js.map +1 -1
- package/dist/types/extract-hast/strongly.d.ts +2 -1
- package/dist/types/extract-hast/strongly.js.map +1 -1
- package/dist/types/extract-hast/utils.d.ts +0 -1
- package/dist/types/extract-hast/utils.js.map +1 -1
- package/dist/types/html-to-mdast.d.ts +4 -0
- package/dist/types/html-to-mdast.js.map +1 -1
- package/dist/types/link-replacer.d.ts +1 -1
- package/dist/types/link-replacer.js.map +1 -1
- package/dist/types/loaders/playwright.d.ts +8 -2
- package/dist/types/loaders/playwright.js.map +1 -1
- package/dist/types/mdast-handlers/custom-br-handler.d.ts +2 -0
- package/dist/types/mdast-handlers/custom-br-handler.js.map +1 -0
- package/dist/types/mdast-to-markdown.js.map +1 -1
- package/dist/types/utils/hast-utils.d.ts +4 -0
- package/dist/types/utils/hast-utils.js.map +1 -0
- package/dist/utils/hast-utils.js +46 -0
- package/package.json +22 -10
- package/dist/cjs/cli/bin.js +0 -34
- package/dist/cjs/cli/commands/handlers/index.js +0 -84
- package/dist/cjs/cli/commands/handlers/loadHtml.js +0 -127
- package/dist/cjs/cli/helpers/assertContinue.js +0 -34
- package/dist/cjs/cli/helpers/inputOutputPath.js +0 -89
- package/dist/cjs/cli/helpers/inputSourcePath.js +0 -64
- package/dist/cjs/cli/helpers/selectExtractMode.js +0 -39
- package/dist/cjs/cli/helpers/selectLoader.js +0 -44
- package/dist/cjs/cli/utils.js +0 -122
- package/dist/cjs/html-to-markdown.test.js +0 -118
- package/dist/cjs/link-replacer.test.js +0 -47
- package/dist/cjs/loaders/fetch.test.js +0 -9
- package/dist/cjs/loaders/playwright.test.js +0 -17
- package/dist/cjs/loaders/puppeteer.test.js +0 -17
- package/dist/cli/bin.js +0 -12
- package/dist/cli/commands/handlers/index.js +0 -51
- package/dist/cli/commands/handlers/loadHtml.js +0 -94
- package/dist/cli/constants.js +0 -9
- package/dist/cli/helpers/assertContinue.js +0 -11
- package/dist/cli/helpers/inputOutputPath.js +0 -56
- package/dist/cli/helpers/inputSourcePath.js +0 -31
- package/dist/cli/helpers/selectExtractMode.js +0 -16
- package/dist/cli/helpers/selectLoader.js +0 -21
- package/dist/cli/utils.js +0 -85
- package/dist/html-to-markdown.test.js +0 -118
- package/dist/link-replacer.test.js +0 -47
- package/dist/loaders/fetch.test.js +0 -9
- package/dist/loaders/playwright.test.js +0 -17
- package/dist/loaders/puppeteer.test.js +0 -17
- package/dist/types/cli/commands/handlers/index.d.ts +0 -13
- package/dist/types/cli/commands/handlers/index.js.map +0 -1
- package/dist/types/cli/commands/handlers/loadHtml.d.ts +0 -2
- package/dist/types/cli/commands/handlers/loadHtml.js.map +0 -1
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
// src/loaders/playwright.test.ts
|
|
2
|
-
import { chromium } from "playwright";
|
|
3
|
-
import { describe, expect, it } from "vitest";
|
|
4
|
-
import { loadHtml } from "./playwright.js";
|
|
5
|
-
describe("Playwright loader", () => {
|
|
6
|
-
it("should load the HTML of a URL", async () => {
|
|
7
|
-
const html = await loadHtml("https://example.com");
|
|
8
|
-
expect(html).toContain("Example Domain");
|
|
9
|
-
});
|
|
10
|
-
it("should load the HTML of a URL using a custom context", async () => {
|
|
11
|
-
const context = await chromium.launch({ headless: true });
|
|
12
|
-
const html = await loadHtml("https://example.com", context);
|
|
13
|
-
expect(html).toContain("Example Domain");
|
|
14
|
-
expect(context.isConnected()).toBe(true);
|
|
15
|
-
await context.close();
|
|
16
|
-
});
|
|
17
|
-
});
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
// src/loaders/puppeteer.test.ts
|
|
2
|
-
import { chromium } from "playwright.js";
|
|
3
|
-
import { describe, expect, it } from "vitest";
|
|
4
|
-
import { loadHtml } from "./puppeteer.js";
|
|
5
|
-
describe("Puppeteer loader", () => {
|
|
6
|
-
it("should load the HTML of a URL", async () => {
|
|
7
|
-
const html = await loadHtml("https://example.com");
|
|
8
|
-
expect(html).toContain("Example Domain");
|
|
9
|
-
});
|
|
10
|
-
it("should load the HTML of a URL using a custom puppeteer context", async () => {
|
|
11
|
-
const context = await chromium.launch({ headless: true });
|
|
12
|
-
const html = await loadHtml("https://example.com", { headless: true });
|
|
13
|
-
expect(html).toContain("Example Domain");
|
|
14
|
-
expect(context.isConnected()).toBe(true);
|
|
15
|
-
await context.close();
|
|
16
|
-
});
|
|
17
|
-
});
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
export type LoaderOptions = {
|
|
2
|
-
type: "web";
|
|
3
|
-
loader: "playwright" | "pw" | "fetch" | string;
|
|
4
|
-
} | {
|
|
5
|
-
type: "local";
|
|
6
|
-
loader: null;
|
|
7
|
-
};
|
|
8
|
-
export declare const webforaiCommand: (initialPath: string, initialOutputPath: string, options: {
|
|
9
|
-
mode?: string;
|
|
10
|
-
loader: string;
|
|
11
|
-
debug?: boolean;
|
|
12
|
-
baseUrl?: string;
|
|
13
|
-
}) => Promise<void>;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/cli/commands/handlers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,MAAM,aAAa,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAC/E,MAAM,sBAAsB,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAM3F,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EACnC,WAAmB,EACnB,iBAAyB,EACzB,OAA6E,EAC5E,EAAE;IACH,MAAM,UAAU,GAAG,WAAW,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;IAC5D,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,MAAM,GAAkB,KAAK,CAAC,UAAU,CAAC;QAC9C,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,YAAY,EAAE,CAAC,EAAE;QACnE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,UAAU,GAAG,iBAAiB,IAAI,CAAC,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC;IACzD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACnD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,EAAE;QACxC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO;QACzD,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,sBAAsB,CAAC;KAC3D,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAE3C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IACD,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEvC,KAAK,CAAC,kCAAkC,CAAC,CAAC;AAC3C,CAAC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"loadHtml.js","sourceRoot":"","sources":["../../../../../src/cli/commands/handlers/loadHtml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,OAAO,IAAI,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACnF,OAAO,EAAE,MAAM,YAAY,CAAC;AAE5B,OAAO,EAAE,CAAC,EAAE,MAAM,IAAI,CAAC;AAKvB,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,UAAkB,EAAE,MAAqB,EAAmB,EAAE;IAC5F,MAAM,WAAW,GAAG,YAAY,EAAE,CAAC;IAEnC,IAAI,CAAC;QACJ,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,OAAO;gBACX,OAAO,MAAM,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;YACrD,KAAK,KAAK;gBACT,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;oBACvB,KAAK,OAAO;wBACX,OAAO,MAAM,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBACrD,KAAK,YAAY;wBAChB,OAAO,MAAM,kBAAkB,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBAC1D;wBACC,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;gBACpC,CAAC;YACF;gBACC,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,CAAC;QACpD,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,KAAK,EAAE,UAAkB,EAAE,WAAwB,EAAmB,EAAE;IAC7F,WAAW,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,IAAI,CAAC;QACJ,MAAM,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACvD,WAAW,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IACnD,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,KAAK,EAAE,UAAkB,EAAE,WAAwB,EAAmB,EAAE;IAC7F,WAAW,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC3C,WAAW,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,KAAK,EAAE,UAAkB,EAAE,WAAwB,EAAmB,EAAE;IAClG,MAAM,aAAa,GAAG,YAAY,EAAE,CAAC;IACrC,aAAa,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IAE7C,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAC9C,IAAI,OAA0B,CAAC;IAE/B,IAAI,CAAC;QACJ,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC7C,aAAa,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,2BAA2B,CAAC,KAAc,CAAC,CAAC;QAClD,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC9C,CAAC;IAED,WAAW,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAElD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAEtB,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,2BAA2B,GAAG,KAAK,EAAE,KAAY,EAAiB,EAAE;IACzE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;QACvD,MAAM,KAAK,CAAC;IACb,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC;QAC/B,OAAO,EAAE,4EAA4E;KACrF,CAAC,CAAC;IACH,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC1B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,kBAAkB,GAAG,OAAO,EAAE,CAAC;IACrC,kBAAkB,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAE7D,IAAI,CAAC;QACJ,MAAM,CAAC,CAAA,wBAAwB,CAAC;QAChC,kBAAkB,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC5D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC,CAAC;QAC3E,MAAM,KAAK,CAAC;IACb,CAAC;AACF,CAAC,CAAC"}
|