smart-web-mcp 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -5
- package/README.md +7 -53
- package/dist/settings.d.ts +0 -6
- package/dist/settings.js +0 -3
- package/dist/settings.js.map +1 -1
- package/dist/smartfetch/providers/index.js +1 -3
- package/dist/smartfetch/providers/index.js.map +1 -1
- package/dist/smartfetch/providers/x.js +192 -5
- package/dist/smartfetch/providers/x.js.map +1 -1
- package/dist/smartfetch.js +7 -1
- package/dist/smartfetch.js.map +1 -1
- package/dist/smartsearch.js +0 -155
- package/dist/smartsearch.js.map +1 -1
- package/package.json +2 -2
- package/dist/smartfetch/providers/firecrawl.d.ts +0 -19
- package/dist/smartfetch/providers/firecrawl.js +0 -117
- package/dist/smartfetch/providers/firecrawl.js.map +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,7 @@ The format is based on Keep a Changelog[^1] and this project uses SemVer.
|
|
|
8
8
|
|
|
9
9
|
### Added
|
|
10
10
|
|
|
11
|
+
- Removed hosted API-key provider lanes (Firecrawl, Exa, Tavily, Browserless) from the smart-web runtime/benchmark surface so the default product remains no-secret/local-first.
|
|
11
12
|
- `smartfetch` now accepts an MCP `output_budget` preset (`compact`, `balanced`, or `full`) and `fetch.outputBudget.preset` settings, so token-sensitive hosts can request smaller structured payloads without hand-tuning every numeric cap.
|
|
12
13
|
- Added MCP host-profile guidance for Hermes/Raon, Codex, OpenCode, and Claude Code, including budget-control precedence and compact/balanced/full follow-up retrieval guidance.
|
|
13
14
|
- Added deterministic MCP smoke coverage proving large `smartfetch` structured payloads stay out of `content[0].text` unless `format: "json"` is explicitly requested.
|
|
@@ -42,13 +43,10 @@ The format is based on Keep a Changelog[^1] and this project uses SemVer.
|
|
|
42
43
|
- Added an explicit `--allow-naver-map-live-smoke` provider benchmark gate and `benchmark:providers:live:naver-map` alias for public Naver Map browser-lane live evidence.
|
|
43
44
|
- Added an explicit `--allow-naver-cafe-live-smoke` provider benchmark gate and `benchmark:providers:live:naver-cafe` alias for public Naver Cafe partial-honesty live evidence.
|
|
44
45
|
- Added `--allow-all-no-secret-live-smokes` and `benchmark:providers:live:no-secret` so maintenance jobs can archive one all-no-secret live calibration artifact without enabling credentialed search lanes.
|
|
45
|
-
- Added an explicit `--allow-external-search-live-smoke` provider benchmark gate and `benchmark:providers:live:external-search` alias for credentialed Exa/Tavily search live-smoke policy calibration.
|
|
46
|
-
- Added a Tavily `smartsearch` runtime provider behind `search.tavilyApiKey` / `search.enableTavily`, plus credentialed external-search live-smoke support when `TAVILY_API_KEY` or `SMART_WEB_TAVILY_API_KEY` is explicitly configured.
|
|
47
46
|
- Added `--json-file <path>` and an npm alias for provider benchmark JSON artifacts without stdout capture.
|
|
48
47
|
- Added `--gap-file <path>` and `benchmark:providers:gaps` for compact provider evidence gap JSON artifacts.
|
|
49
48
|
- Added `benchmark:providers:live:no-secret:gaps` for refreshing the compact all-no-secret live provider evidence gap artifact without archiving the full Markdown/JSON report.
|
|
50
49
|
- Added a provider benchmark report archive index in `reports/README.md` that points maintenance jobs at the current no-secret artifacts and labels older cumulative reports as historical calibration snapshots.
|
|
51
|
-
- Added deterministic Exa/Tavily-style `search_external` provider benchmark fixtures so credentialed search lanes can be measured without paid/live requests before any external-search live-smoke work.
|
|
52
50
|
- Added fixture skipped-case lane/reason metadata to provider benchmark live-vs-fixture and evidence-gap artifacts so credential-gated fixtures are not misread as absent fixtures.
|
|
53
51
|
- Added `scripts/partial-handoff-watchdog.mjs` and `npm run watchdog:partial-handoff` to run a no-secret drift watchdog over representative partial/blocked/handoff outputs and fail on chrome URL, script, or style-shell leakage.
|
|
54
52
|
- Added `--report-file <path>` to the partial/handoff watchdog so durable runners can persist the full JSON report while keeping stdout human-readable.
|
|
@@ -60,6 +58,8 @@ The format is based on Keep a Changelog[^1] and this project uses SemVer.
|
|
|
60
58
|
- Added Threads unavailable/join-login output to the partial/handoff watchdog representative set.
|
|
61
59
|
- Added Instagram unavailable/login output to the partial/handoff watchdog representative set.
|
|
62
60
|
- Added X/Twitter and Reddit unavailable/authwall outputs to the partial/handoff watchdog representative set.
|
|
61
|
+
- Added no-secret X/Twitter profile/timeline syndication handling with explicit blocked/rate-limited guidance when the public endpoint is unavailable.
|
|
62
|
+
- Added deterministic and gated live provider benchmark coverage for public X/Twitter post retrieval, including `--allow-x-twitter-live-smoke` and `benchmark:providers:live:x-twitter`.
|
|
63
63
|
- Added DCInside unavailable/deleted post output to the partial/handoff watchdog representative set.
|
|
64
64
|
- Added Telegram missing/private post output to the partial/handoff watchdog representative set.
|
|
65
65
|
- Added an explicit `smartcrawl-export.v1` schema version to smartcrawl export manifests and summaries.
|
|
@@ -67,12 +67,13 @@ The format is based on Keep a Changelog[^1] and this project uses SemVer.
|
|
|
67
67
|
- Added resolved crawl options to smartcrawl export manifests for reproducible docs exports.
|
|
68
68
|
- Added versioned `smartcrawl-resource-handles.json` files to smartcrawl docs exports so hosts can consume local Markdown file handles without parsing the full crawl manifest.
|
|
69
69
|
- Added ordered `recommended_resources` to `smartcrawl-resource-handles.json`, pointing hosts at the summary, resource-handle manifest, and first Markdown document without requiring role-order heuristics.
|
|
70
|
-
- Expanded provider benchmark corpus coverage across static articles, JS-rendered pages, authwalls, long docs/Wikipedia, Naver Blog/Cafe/Map, Hacker News, YouTube, BOJ/Codeforces, relay
|
|
70
|
+
- Expanded provider benchmark corpus coverage across static articles, JS-rendered pages, authwalls, long docs/Wikipedia, Naver Blog/Cafe/Map, Hacker News, YouTube, BOJ/Codeforces, relay and docs export surfaces.
|
|
71
71
|
|
|
72
72
|
### Changed
|
|
73
73
|
|
|
74
74
|
- Provider benchmark live-vs-fixture and evidence-gap artifacts now mark all-skipped fixture rows as `fixtureEvidence: "skipped"` with `fixture-skipped-*` comparison states, while still using `unlock-skipped-fixture-prerequisites` for credential-gated follow-up queues.
|
|
75
75
|
- The partial/handoff watchdog now uses a non-existent Wikidocs article id for its unavailable-shell representative, avoiding false failures when public Wikidocs articles render as healthy content.
|
|
76
|
+
- X/Twitter oEmbed fallback output now includes available author/date metadata and suppresses `ref_src` tracking links from compact post links.
|
|
76
77
|
- `smartcrawl` export summaries now include file:// re-open guidance for the summary, manifest, resource-handle JSON, and first Markdown document so agents can resume from export handles without requesting JSON duplication.
|
|
77
78
|
- `smartcrawl-resource-handles.json` now includes metadata handles for the export summary, crawl manifest, and resource-handle manifest alongside document handles.
|
|
78
79
|
- `smartcrawl` export error summaries now label unwritten summary/manifest/resource-handle paths as planned paths and include retry guidance instead of presenting them as re-openable files.
|
|
@@ -139,6 +140,20 @@ The format is based on Keep a Changelog[^1] and this project uses SemVer.
|
|
|
139
140
|
- Korean public API helpers now validate outbound URLs and redirects before fetching, blocking localhost, private, and reserved targets unless `network.allowPrivateHosts` is explicitly enabled.
|
|
140
141
|
- CI now declares read-only default repository permissions, Dependabot auto-merge pins the reusable workflow without forwarding all caller secrets, and the workflow check enforces those guards.
|
|
141
142
|
|
|
143
|
+
## [0.29.1](https://github.com/jojo-labs/smart-web/compare/v0.29.0...v0.29.1) (2026-06-22)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
### Refactoring
|
|
147
|
+
|
|
148
|
+
* remove hosted provider lanes ([#362](https://github.com/jojo-labs/smart-web/issues/362)) ([b1871a9](https://github.com/jojo-labs/smart-web/commit/b1871a953305612cb2df63600d6341c099ac3131))
|
|
149
|
+
|
|
150
|
+
## [0.29.0](https://github.com/jojo-labs/smart-web/compare/v0.28.0...v0.29.0) (2026-06-21)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
### Features
|
|
154
|
+
|
|
155
|
+
* improve X profile and oEmbed fetches ([#360](https://github.com/jojo-labs/smart-web/issues/360)) ([d75ac91](https://github.com/jojo-labs/smart-web/commit/d75ac91f1da52977841affeacf5f724ab7762b20))
|
|
156
|
+
|
|
142
157
|
## [0.28.0](https://github.com/jojo-labs/smart-web/compare/v0.27.0...v0.28.0) (2026-06-21)
|
|
143
158
|
|
|
144
159
|
|
|
@@ -1209,7 +1224,7 @@ The format is based on Keep a Changelog[^1] and this project uses SemVer.
|
|
|
1209
1224
|
### Added
|
|
1210
1225
|
|
|
1211
1226
|
- initial public release of `smart-web-mcp`
|
|
1212
|
-
- `smartsearch` with
|
|
1227
|
+
- `smartsearch` with site-native, SearXNG, DuckDuckGo, and Brave HTML fallback chain
|
|
1213
1228
|
- `smartfetch` with impit and Playwright escalation
|
|
1214
1229
|
- Reddit, X, and DCInside target-specific handling
|
|
1215
1230
|
- npm publication and MCP Registry publication
|
package/README.md
CHANGED
|
@@ -236,10 +236,6 @@ npx -y smart-web-mcp --init-settings
|
|
|
236
236
|
},
|
|
237
237
|
|
|
238
238
|
"search": {
|
|
239
|
-
// API keys — leave empty to skip that provider
|
|
240
|
-
"exaApiKey": "",
|
|
241
|
-
"tavilyApiKey": "",
|
|
242
|
-
"braveSearchApiKey": "",
|
|
243
239
|
// Self-hosted SearXNG instance URL
|
|
244
240
|
"searxngBaseUrl": "",
|
|
245
241
|
"enableSearxng": true
|
|
@@ -336,44 +332,7 @@ Out of the box `smart-web` works with sensible defaults. Create a settings file
|
|
|
336
332
|
}
|
|
337
333
|
```
|
|
338
334
|
|
|
339
|
-
In `private` mode, relay-style provider requests are blocked,
|
|
340
|
-
|
|
341
|
-
#### Private with Exa allowed back in
|
|
342
|
-
|
|
343
|
-
```json
|
|
344
|
-
{
|
|
345
|
-
"profile": "private",
|
|
346
|
-
"search": {
|
|
347
|
-
"exaApiKey": "...",
|
|
348
|
-
"enableExa": true
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
```
|
|
352
|
-
|
|
353
|
-
#### Private with Tavily allowed back in
|
|
354
|
-
|
|
355
|
-
```json
|
|
356
|
-
{
|
|
357
|
-
"profile": "private",
|
|
358
|
-
"search": {
|
|
359
|
-
"tavilyApiKey": "...",
|
|
360
|
-
"enableTavily": true
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
```
|
|
364
|
-
|
|
365
|
-
#### With Exa, Tavily, and Brave API keys
|
|
366
|
-
|
|
367
|
-
```json
|
|
368
|
-
{
|
|
369
|
-
"profile": "balanced",
|
|
370
|
-
"search": {
|
|
371
|
-
"exaApiKey": "exa-...",
|
|
372
|
-
"tavilyApiKey": "tvly-...",
|
|
373
|
-
"braveSearchApiKey": "BSA-..."
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
```
|
|
335
|
+
In `private` mode, relay-style provider requests are blocked, public no-key search fallbacks are disabled, and SearXNG bases must resolve to localhost or private addresses. Hosted API-key search providers such as Exa, Tavily, and Brave Search are intentionally not part of the default smartsearch runtime; use local/site-native search, DuckDuckGo/Brave HTML fallbacks, or self-hosted SearXNG instead.
|
|
377
336
|
|
|
378
337
|
#### Disable Playwright auto-install
|
|
379
338
|
|
|
@@ -469,9 +428,6 @@ These keys live inside `settings.json` when you need to force one provider on or
|
|
|
469
428
|
|
|
470
429
|
**Search**
|
|
471
430
|
|
|
472
|
-
- `search.enableExa`
|
|
473
|
-
- `search.enableTavily`
|
|
474
|
-
- `search.enableBrave`
|
|
475
431
|
- `search.enableSearxng`
|
|
476
432
|
- `search.enableDuckDuckGo`
|
|
477
433
|
- `search.enableBraveHtml`
|
|
@@ -534,26 +490,24 @@ npm run benchmark:providers:live:blocked
|
|
|
534
490
|
npm run benchmark:providers:live:naver-blog
|
|
535
491
|
npm run benchmark:providers:live:naver-map
|
|
536
492
|
npm run benchmark:providers:live:naver-cafe
|
|
493
|
+
npm run benchmark:providers:live:x-twitter
|
|
537
494
|
npm run benchmark:providers:live:no-secret
|
|
538
495
|
npm run benchmark:providers:live:no-secret:gaps
|
|
539
|
-
npm run benchmark:providers:live:external-search
|
|
540
496
|
npm run benchmark:providers:trend
|
|
541
497
|
npm run watchdog:partial-handoff
|
|
542
498
|
```
|
|
543
499
|
|
|
544
|
-
The default report is availability-only and safe for CI. `--run-measurements` (`npm run benchmark:providers:measure`) adds deterministic fixture records for available local/core
|
|
500
|
+
The default report is availability-only and safe for CI. `--run-measurements` (`npm run benchmark:providers:measure`) adds deterministic fixture records for available local/core and optional no-key lanes: success, useful character counts, projected/text character counts, useful-to-projected and useful-to-text yield ratios, latency, extractor/provider id, confidence, partial status, error class, lane label/family/category metadata, provider-family/category lane-status rollups, surface coverage, and recommendation sections grouped by lane, case, and surface.
|
|
545
501
|
|
|
546
502
|
Reports carry the `provider-benchmark.v1` schema version plus non-fatal git and runtime metadata so archived JSON, Markdown, and NDJSON trend artifacts are parseable, traceable, and comparable as the harness evolves. Markdown, text, JSON, and NDJSON outputs include aggregate fixture yield averages, compact yield rollups by provider family/category/lane, and a best-current-evidence surface table that ranks measured/skipped candidates so review jobs can compare token efficiency and build-vs-wrap posture between core, browser, relay, and specialist paths without reprocessing the full measurement table.
|
|
547
503
|
|
|
548
|
-
Measurement artifacts also include a live-readiness manifest listing each lane's env/command/settings prerequisites, a deterministic fixture smoke, a safe MCP live-smoke candidate, and surfaces that still lack live evidence. `--run-live-smokes` (`npm run benchmark:providers:live`) is an explicit gated mode that runs only no-secret, core-surface live checks for static articles, long docs/Wikipedia, Hacker News, and docs export;
|
|
549
|
-
|
|
550
|
-
Live smoke artifacts record per-case `present`/`missing`/`skipped` evidence next to fixture evidence without printing secret values. When fixture rows exist but all are skipped by missing credentials or commands, comparison and gap artifacts report fixture evidence as `skipped` with a `fixture-skipped-*` comparison instead of implying the fixture rows are absent. Browser-lane live smoke rows also include a secret-safe Playwright preflight diagnostic with executable path, launch timing, `domcontentloaded` navigation timing, HTTP status/final URL, and truncated failure reason so runtime/cache/network failures can be separated from smartfetch normalization failures.
|
|
504
|
+
Measurement artifacts also include a live-readiness manifest listing each lane's env/command/settings prerequisites, a deterministic fixture smoke, a safe MCP live-smoke candidate, and surfaces that still lack live evidence. `--run-live-smokes` (`npm run benchmark:providers:live`) is an explicit gated mode that runs only no-secret, core-surface live checks for static articles, long docs/Wikipedia, Hacker News, and docs export; optional no-secret lanes remain gated until explicitly allowlisted: `--allow-jina-live-smoke` (`npm run benchmark:providers:live:jina`) exercises the public Jina Reader relay with a secret-stripped settings file, `--allow-competitive-programming-live-smoke` (`npm run benchmark:providers:live:cp`) adds a public BOJ/Codeforces representative smoke for the competitive-programming specialist surface, `--allow-browser-live-smoke` (`npm run benchmark:providers:live:browser`) runs a Playwright `--force-dynamic` smoke for the JS-rendered article surface only when the local browser lane is available, `--allow-youtube-live-smoke` (`npm run benchmark:providers:live:youtube`) adds the public YouTube seed-lane representative smoke, `--allow-blocked-authwall-live-smoke` (`npm run benchmark:providers:live:blocked`) adds a deterministic HTTP Basic Auth challenge smoke that treats honest blocked/error/handoff signals as positive calibration without private credentials, `--allow-naver-blog-live-smoke` (`npm run benchmark:providers:live:naver-blog`) adds a stable public Naver Blog representative smoke for Korean specialist-lane calibration, `--allow-naver-map-live-smoke` (`npm run benchmark:providers:live:naver-map`) adds a stable public Naver Map place smoke with browser-preflight diagnostics, `--allow-naver-cafe-live-smoke` (`npm run benchmark:providers:live:naver-cafe`) adds a stable public Naver Cafe shared-link smoke that treats useful no-credential partial output as honest calibration, and `--allow-x-twitter-live-smoke` (`npm run benchmark:providers:live:x-twitter`) adds a public X/Twitter post smoke that keeps no-secret social-provider drift visible without private cookies. `--allow-all-no-secret-live-smokes` (`npm run benchmark:providers:live:no-secret`) enables every no-secret gate in one run; hosted API-key search lanes are excluded from the benchmark surface by product policy.
|
|
551
505
|
|
|
552
|
-
|
|
506
|
+
Live smoke artifacts record per-case `present`/`missing`/`skipped` evidence next to fixture evidence without printing secret values. When fixture rows exist but all are skipped by missing local commands, comparison and gap artifacts report fixture evidence as `skipped` with a `fixture-skipped-*` comparison instead of implying the fixture rows are absent. Browser-lane live smoke rows also include a secret-safe Playwright preflight diagnostic with executable path, launch timing, `domcontentloaded` navigation timing, HTTP status/final URL, and truncated failure reason so runtime/cache/network failures can be separated from smartfetch normalization failures.
|
|
553
507
|
|
|
554
|
-
The same JSON, text, Markdown, and NDJSON outputs include a compact live-vs-fixture comparison rollup so replacement reviews can distinguish calibrated surfaces from fixture-only, skipped-live, live-disagreeing, and fixture-skipped-by-prerequisite surfaces. Secret-safe provider diagnostics list prerequisite names with satisfied/missing status and never print env values. Use `npm run benchmark:providers:json-file` or add `--json-file <path>` to write the full JSON artifact without relying on stdout capture. Use `npm run benchmark:providers:report` or add `--report-file <path>` with measurement mode to write a Markdown review artifact containing lane availability, diagnostics, surface/corpus coverage, measurement highlights, yield comparisons, per-surface evidence, live readiness, live-smoke diagnostics, and build-vs-wrap recommendations without changing stdout or JSON behavior. Use `npm run benchmark:providers:gaps` or add `--gap-file <path>` with measurement mode to write a compact `provider-benchmark-gaps.v1` JSON artifact containing only the ranked provider evidence gap queue, summary counts, git/runtime metadata, and safe smoke candidates. Use `npm run benchmark:providers:trend` or add `--trend-file <path>` with measurement mode to append compact NDJSON trend records with aggregate lane, prerequisite, measurement, recommendation, surface-evidence, and fixture-yield counts. Unknown `--flag` options fail fast so typoed benchmark jobs do not produce misleading artifacts. The corpus covers static articles, JS-rendered articles, blocked/authwall-like pages, long docs/Wikipedia pages, Naver Blog/Cafe/Map, Hacker News, YouTube, BOJ/Codeforces, relay,
|
|
508
|
+
The same JSON, text, Markdown, and NDJSON outputs include a compact live-vs-fixture comparison rollup so replacement reviews can distinguish calibrated surfaces from fixture-only, skipped-live, live-disagreeing, and fixture-skipped-by-prerequisite surfaces. Secret-safe provider diagnostics list prerequisite names with satisfied/missing status and never print env values. Use `npm run benchmark:providers:json-file` or add `--json-file <path>` to write the full JSON artifact without relying on stdout capture. Use `npm run benchmark:providers:report` or add `--report-file <path>` with measurement mode to write a Markdown review artifact containing lane availability, diagnostics, surface/corpus coverage, measurement highlights, yield comparisons, per-surface evidence, live readiness, live-smoke diagnostics, and build-vs-wrap recommendations without changing stdout or JSON behavior. Use `npm run benchmark:providers:gaps` or add `--gap-file <path>` with measurement mode to write a compact `provider-benchmark-gaps.v1` JSON artifact containing only the ranked provider evidence gap queue, summary counts, git/runtime metadata, and safe smoke candidates. Use `npm run benchmark:providers:trend` or add `--trend-file <path>` with measurement mode to append compact NDJSON trend records with aggregate lane, prerequisite, measurement, recommendation, surface-evidence, and fixture-yield counts. Unknown `--flag` options fail fast so typoed benchmark jobs do not produce misleading artifacts. The corpus covers static articles, JS-rendered articles, blocked/authwall-like pages, long docs/Wikipedia pages, Naver Blog/Cafe/Map, Hacker News, YouTube, X/Twitter, BOJ/Codeforces, relay, and docs export surfaces. Optional local/no-secret lanes without commands remain explicit skips instead of silently failing; API-key provider candidates are excluded from the default corpus and reports by product policy.
|
|
555
509
|
|
|
556
|
-
Provider benchmark reports also include `providerEvidenceGaps`: a ranked next-action queue built from the live-vs-fixture rollup. Each gap records the surface, priority score, selected and candidate lanes, follow-up action, unlock prerequisites, fixture/live useful chars, fixture skipped-case lanes/reasons, live-smoke failure error classes/reasons, and safe live smoke candidate so replacement reviews can pick the next provider experiment without scanning every measurement row or re-running a timed live calibration. Use `npm run benchmark:providers:live:no-secret:gaps` to refresh only the compact all-no-secret gap artifact at `reports/provider-benchmark-live-no-secret-gaps.json`. When fixture rows exist but were skipped by missing
|
|
510
|
+
Provider benchmark reports also include `providerEvidenceGaps`: a ranked next-action queue built from the live-vs-fixture rollup. Each gap records the surface, priority score, selected and candidate lanes, follow-up action, unlock prerequisites, fixture/live useful chars, fixture skipped-case lanes/reasons, live-smoke failure error classes/reasons, and safe live smoke candidate so replacement reviews can pick the next provider experiment without scanning every measurement row or re-running a timed live calibration. Use `npm run benchmark:providers:live:no-secret:gaps` to refresh only the compact all-no-secret gap artifact at `reports/provider-benchmark-live-no-secret-gaps.json`. When fixture rows exist but were skipped by missing local commands, the live-vs-fixture and gap actions are `unlock-skipped-fixture-prerequisites` rather than fixture-creation actions.
|
|
557
511
|
|
|
558
512
|
See `reports/README.md` for the curated report archive index. It identifies `provider-benchmark-live-no-secret.{json,md}` and `provider-benchmark-live-no-secret-gaps.json` as the current maintenance artifacts, and labels older cumulative live-smoke reports as historical calibration snapshots.
|
|
559
513
|
|
package/dist/settings.d.ts
CHANGED
|
@@ -5,13 +5,7 @@ export type SmartWebSettings = {
|
|
|
5
5
|
tempDir?: string;
|
|
6
6
|
};
|
|
7
7
|
search?: {
|
|
8
|
-
exaApiKey?: string;
|
|
9
|
-
tavilyApiKey?: string;
|
|
10
|
-
braveSearchApiKey?: string;
|
|
11
8
|
searxngBaseUrl?: string;
|
|
12
|
-
enableExa?: boolean;
|
|
13
|
-
enableTavily?: boolean;
|
|
14
|
-
enableBrave?: boolean;
|
|
15
9
|
enableSearxng?: boolean;
|
|
16
10
|
enableDuckDuckGo?: boolean;
|
|
17
11
|
enableBraveHtml?: boolean;
|
package/dist/settings.js
CHANGED
package/dist/settings.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"settings.js","sourceRoot":"","sources":["../src/settings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC/G,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAA;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"settings.js","sourceRoot":"","sources":["../src/settings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC/G,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAA;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAgElD,IAAI,KAAK,GAAiC,SAAS,CAAA;AACnD,IAAI,OAAO,GAAG,EAAE,CAAA;AAChB,IAAI,sBAAsB,GAAuB,SAAS,CAAA;AAE1D,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,SAAS;IACT,UAAU;IACV,UAAU;IACV,aAAa;CACd,CAAA;AAED,SAAS,iBAAiB;IACxB,MAAM,IAAI,GAAG,OAAO,EAAE,CAAA;IACtB,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;IACzE,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,qBAAqB,CAAC,CAAA;IACtF,OAAO,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;AAC9B,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAa;IACjD,sBAAsB,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IACzD,KAAK,GAAG,SAAS,CAAA;IACjB,OAAO,GAAG,EAAE,CAAA;AACd,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,IAAI,sBAAsB;QAAE,OAAO,sBAAsB,CAAA;IACzD,OAAO,IAAI,CAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,eAAe,CAAC,CAAA;AAChE,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,OAAO,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAyB,CAAC,CAAC,CAAC,EAAE,CAAA;AACrG,CAAC;AAED,SAAS,YAAY;IACnB,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAA;IACrC,MAAM,IAAI,GAAG,YAAY,EAAE,CAAA;IAC3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,KAAK,GAAG,EAAE,CAAA;QACV,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,CAAC;QACH,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;IAChE,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACtE,MAAM,IAAI,GAAG,qCAAqC,IAAI,KAAK,OAAO,EAAE,CAAA;QACpE,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACzC,OAAO,GAAG,IAAI,CAAA;QACd,KAAK,GAAG,EAAE,CAAA;IACZ,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,IAAI,KAAK,GAAY,YAAY,EAAE,CAAA;IACnC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACtC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC;YAAE,OAAO,SAAS,CAAA;QACxG,KAAK,GAAI,KAAiC,CAAC,OAAO,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAA;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAA;IAClD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAA;IACjF,OAAO,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,OAAO;QACL,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE;YACP,OAAO,EAAE,EAAE;SACZ;QACD,MAAM,EAAE;YACN,cAAc,EAAE,EAAE;YAClB,aAAa,EAAE,IAAI;SACpB;QACD,KAAK,EAAE;YACL,aAAa,EAAE,EAAE;YACjB,UAAU,EAAE,EAAE;YACd,qBAAqB,EAAE,IAAI;YAC3B,gBAAgB,EAAE,KAAK;YACvB,iBAAiB,EAAE,oBAAoB;YACvC,sBAAsB,EAAE,IAAI;YAC5B,cAAc,EAAE,IAAI;YACpB,eAAe,EAAE,IAAI;YACrB,gBAAgB,EAAE,IAAI;YACtB,eAAe,EAAE,IAAI;YACrB,cAAc,EAAE,EAAE;YAClB,qBAAqB,EAAE,IAAI;YAC3B,qBAAqB,EAAE,EAAE;YACzB,mBAAmB,EAAE,IAAI;YACzB,UAAU,EAAE,EAAE;YACd,eAAe,EAAE,IAAI;YACrB,4BAA4B,EAAE,IAAI;YAClC,4BAA4B,EAAE,SAAS;YACvC,gBAAgB,EAAE,IAAI;YACtB,uBAAuB,EAAE,IAAI;YAC7B,qBAAqB,EAAE,IAAI;YAC3B,aAAa,EAAE,IAAI;YACnB,eAAe,EAAE,IAAI;YACrB,YAAY,EAAE;gBACZ,MAAM,EAAE,UAAU;gBAClB,gBAAgB,EAAE,KAAK;gBACvB,oBAAoB,EAAE,KAAK;gBAC3B,cAAc,EAAE,EAAE;gBAClB,eAAe,EAAE,EAAE;gBACnB,gBAAgB,EAAE,GAAG;gBACrB,SAAS,EAAE,EAAE;gBACb,SAAS,EAAE,EAAE;gBACb,iBAAiB,EAAE,IAAI;gBACvB,gBAAgB,EAAE,IAAI;aACvB;YACD,cAAc,EAAE,EAAE;SACnB;QACD,OAAO,EAAE;YACP,iBAAiB,EAAE,KAAK;SACzB;KACF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAI,GAAG,YAAY,EAAE,EAAE,KAAK,GAAG,KAAK;IAC/D,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC1B,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;IAC/D,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACzB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;IAChD,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IACrB,MAAM,OAAO,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,MAAM,CAAA;IAC5C,YAAY,CAAC,aAAa,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;IAC1H,IAAI,CAAC;QACH,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC;YACH,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;QAClC,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QACV,MAAM,KAAK,CAAA;IACb,CAAC;IACD,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACnC,qBAAqB,CAAC,IAAI,CAAC,CAAA;IAC3B,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;AAChC,CAAC;AAED,MAAM,UAAU,uBAAuB;IACrC,KAAK,GAAG,SAAS,CAAA;IACjB,OAAO,GAAG,EAAE,CAAA;AACd,CAAC"}
|
|
@@ -6,7 +6,6 @@ import { atcoderProvider } from "./atcoder.js";
|
|
|
6
6
|
import { commerceProvider } from "./commerce.js";
|
|
7
7
|
import { codeforcesProvider } from "./codeforces.js";
|
|
8
8
|
import { dcinsideProvider } from "./dcinside.js";
|
|
9
|
-
import { firecrawlProvider } from "./firecrawl.js";
|
|
10
9
|
import { genericProvider } from "./generic.js";
|
|
11
10
|
import { hackernewsProvider } from "./hackernews.js";
|
|
12
11
|
import { instagramProvider } from "./instagram.js";
|
|
@@ -30,7 +29,7 @@ import { wikipediaProvider } from "./wikipedia.js";
|
|
|
30
29
|
import { wikidocsProvider } from "./wikidocs.js";
|
|
31
30
|
import { xProvider } from "./x.js";
|
|
32
31
|
import { youtubeProvider } from "./youtube.js";
|
|
33
|
-
const providers = [hackernewsProvider, redditProvider, xProvider, threadsProvider, instagramProvider, telegramProvider, dcinsideProvider, youtubeProvider, naverMapProvider, naverMapSearchProvider, kakaoMapProvider, naverBlogProvider, naverCafeProvider, tistoryProvider, velogProvider, linkedinProvider, algumonProvider, namuwikiProvider, wikipediaProvider, wikidocsProvider, solvedacProvider, acmicpcProvider, codeforcesProvider, atcoderProvider, qojProvider, jungolProvider, arxivProvider, archiveProvider, sciencedirectProvider, commerceProvider,
|
|
32
|
+
const providers = [hackernewsProvider, redditProvider, xProvider, threadsProvider, instagramProvider, telegramProvider, dcinsideProvider, youtubeProvider, naverMapProvider, naverMapSearchProvider, kakaoMapProvider, naverBlogProvider, naverCafeProvider, tistoryProvider, velogProvider, linkedinProvider, algumonProvider, namuwikiProvider, wikipediaProvider, wikidocsProvider, solvedacProvider, acmicpcProvider, codeforcesProvider, atcoderProvider, qojProvider, jungolProvider, arxivProvider, archiveProvider, sciencedirectProvider, commerceProvider, genericProvider];
|
|
34
33
|
const providerDefaults = {
|
|
35
34
|
hackernews: { targetFamily: "hackernews_post", priority: 90, capabilities: ["community_post", "supports_assets"], policy: { crossOriginMode: "allow" } },
|
|
36
35
|
reddit: { targetFamily: "reddit_post", priority: 90, capabilities: ["community_post", "supports_assets", "canonical_url"], policy: { crossOriginMode: "allow", preserveCanonicalUrl: true } },
|
|
@@ -62,7 +61,6 @@ const providerDefaults = {
|
|
|
62
61
|
archive: { targetFamily: "generic", priority: 84, capabilities: ["article_extract", "canonical_url"], policy: { preserveCanonicalUrl: true, allowJinaReader: true, allowArchiveFallback: true } },
|
|
63
62
|
sciencedirect: { targetFamily: "generic", priority: 84, capabilities: ["article_extract", "supports_assets", "canonical_url"], policy: { preserveCanonicalUrl: true } },
|
|
64
63
|
commerce: { targetFamily: "generic", priority: 80, capabilities: ["commerce_catalog", "requires_browser", "interactive_surface", "supports_assets"], policy: { crossOriginMode: "same_origin" } },
|
|
65
|
-
firecrawl: { targetFamily: "generic", priority: 15, capabilities: ["article_extract", "supports_assets"], policy: { crossOriginMode: "allow" } },
|
|
66
64
|
generic: { targetFamily: "generic", priority: 10, capabilities: ["article_extract", "supports_assets"], policy: { allowJinaReader: true, allowArchiveFallback: true } },
|
|
67
65
|
};
|
|
68
66
|
function withProviderDefaults(provider) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/smartfetch/providers/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/smartfetch/providers/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACjD,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAA;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAA;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAE9C,MAAM,SAAS,GAAyB,CAAC,kBAAkB,EAAE,cAAc,EAAE,SAAS,EAAE,eAAe,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,eAAe,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,aAAa,EAAE,gBAAgB,EAAE,eAAe,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,eAAe,EAAE,kBAAkB,EAAE,eAAe,EAAE,WAAW,EAAE,cAAc,EAAE,aAAa,EAAE,eAAe,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,eAAe,CAAC,CAAA;AAE3kB,MAAM,gBAAgB,GAA8G;IAClI,UAAU,EAAE,EAAE,YAAY,EAAE,iBAAiB,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,EAAE;IACxJ,MAAM,EAAE,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,gBAAgB,EAAE,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAC7L,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,gBAAgB,EAAE,eAAe,EAAE,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IACpM,OAAO,EAAE,EAAE,YAAY,EAAE,cAAc,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,gBAAgB,EAAE,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAC/L,SAAS,EAAE,EAAE,YAAY,EAAE,gBAAgB,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,gBAAgB,EAAE,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IACnM,QAAQ,EAAE,EAAE,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,gBAAgB,EAAE,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IACjM,QAAQ,EAAE,EAAE,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,aAAa,EAAE,EAAE;IAC3J,OAAO,EAAE,EAAE,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,eAAe,EAAE,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAC/L,WAAW,EAAE,EAAE,YAAY,EAAE,iBAAiB,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,kBAAkB,EAAE,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAC9K,kBAAkB,EAAE,EAAE,YAAY,EAAE,kBAAkB,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,kBAAkB,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,aAAa,EAAE,EAAE;IACtJ,WAAW,EAAE,EAAE,YAAY,EAAE,iBAAiB,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,kBAAkB,EAAE,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAC9K,YAAY,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,kBAAkB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,aAAa,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAC1M,YAAY,EAAE,EAAE,YAAY,EAAE,iBAAiB,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,gBAAgB,EAAE,kBAAkB,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,aAAa,EAAE,cAAc,EAAE,IAAI,EAAE,EAAE;IACtM,OAAO,EAAE,EAAE,YAAY,EAAE,cAAc,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IACtM,KAAK,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAClM,QAAQ,EAAE,EAAE,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,gBAAgB,EAAE,aAAa,EAAE,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,aAAa,EAAE,cAAc,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAC5O,OAAO,EAAE,EAAE,YAAY,EAAE,cAAc,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,gBAAgB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAClJ,QAAQ,EAAE,EAAE,YAAY,EAAE,kBAAkB,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAC3M,SAAS,EAAE,EAAE,YAAY,EAAE,mBAAmB,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAC7M,QAAQ,EAAE,EAAE,YAAY,EAAE,kBAAkB,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAC3M,QAAQ,EAAE,EAAE,YAAY,EAAE,kBAAkB,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,eAAe,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IACtJ,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IACjM,UAAU,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IACpM,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IACjM,GAAG,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAC7L,MAAM,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAChM,KAAK,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IAClN,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IACjM,aAAa,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;IACvK,QAAQ,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,iBAAiB,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,aAAa,EAAE,EAAE;IACjM,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,EAAE;CACxK,CAAA;AAED,SAAS,oBAAoB,CAAC,QAA4B;IACxD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,gBAAgB,CAAC,OAAQ,CAAA;IAC3E,OAAO;QACL,GAAG,QAAQ;QACX,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,QAAQ,CAAC,YAAY;QAC5D,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;QAChD,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,QAAQ,CAAC,YAAY;QAC5D,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM;KAC3C,CAAA;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB;IACrC,OAAO,SAAS,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAA;AAClG,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,GAAW,EAAE,MAAsB;IAC3E,MAAM,OAAO,GAAG,uBAAuB,EAAE,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAA;IAC3F,OAAO,OAAO,IAAI,oBAAoB,CAAC,eAAe,CAAC,CAAA;AACzD,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { asNumber, asString, configEnabled, dedupeUrls, extractUrls, stripTags } from "../../shared.js";
|
|
2
|
-
import { fetchProviderJson } from "../provider-policy.js";
|
|
1
|
+
import { asNumber, asString, configEnabled, decodeHtml, dedupeUrls, extractUrls, stripTags } from "../../shared.js";
|
|
2
|
+
import { fetchProviderJson, fetchProviderText } from "../provider-policy.js";
|
|
3
3
|
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36";
|
|
4
4
|
function localOnlyMode() {
|
|
5
5
|
return configEnabled("network.localOnly", false, { private: true });
|
|
@@ -13,6 +13,10 @@ function allowXOembed() {
|
|
|
13
13
|
const defaultValue = !localOnlyMode();
|
|
14
14
|
return configEnabled("fetch.enableXOembed", defaultValue);
|
|
15
15
|
}
|
|
16
|
+
function allowXSyndication() {
|
|
17
|
+
const defaultValue = !localOnlyMode();
|
|
18
|
+
return configEnabled("fetch.enableXSyndication", defaultValue);
|
|
19
|
+
}
|
|
16
20
|
function extractXStatus(url) {
|
|
17
21
|
try {
|
|
18
22
|
const parsed = new URL(url);
|
|
@@ -30,6 +34,36 @@ function extractXStatus(url) {
|
|
|
30
34
|
}
|
|
31
35
|
return null;
|
|
32
36
|
}
|
|
37
|
+
function extractXScreenNameFromUrl(url) {
|
|
38
|
+
try {
|
|
39
|
+
const parsed = new URL(url);
|
|
40
|
+
const parts = parsed.pathname.split("/").filter(Boolean);
|
|
41
|
+
const handle = asString(parts[0]).replace(/^@/, "").trim();
|
|
42
|
+
return handle && /^[A-Za-z0-9_]{1,15}$/.test(handle) ? handle : "";
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
return "";
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
function extractXProfile(url) {
|
|
49
|
+
try {
|
|
50
|
+
const parsed = new URL(url);
|
|
51
|
+
const host = parsed.hostname.toLowerCase();
|
|
52
|
+
if (!(host === "x.com" || host.endsWith(".x.com") || host === "twitter.com" || host.endsWith(".twitter.com")))
|
|
53
|
+
return null;
|
|
54
|
+
const parts = parsed.pathname.split("/").filter(Boolean);
|
|
55
|
+
if (parts.length !== 1)
|
|
56
|
+
return null;
|
|
57
|
+
const handle = asString(parts[0]).replace(/^@/, "").trim();
|
|
58
|
+
const reserved = new Set(["home", "explore", "search", "notifications", "messages", "settings", "share", "intent", "i"]);
|
|
59
|
+
if (!handle || reserved.has(handle.toLowerCase()) || !/^[A-Za-z0-9_]{1,15}$/.test(handle))
|
|
60
|
+
return null;
|
|
61
|
+
return { screenName: handle };
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
33
67
|
async function fetchFxTwitterStatus(url, timeoutMs) {
|
|
34
68
|
if (!allowFxTwitterRelay())
|
|
35
69
|
return null;
|
|
@@ -56,6 +90,25 @@ async function fetchFxTwitterStatus(url, timeoutMs) {
|
|
|
56
90
|
}
|
|
57
91
|
return null;
|
|
58
92
|
}
|
|
93
|
+
async function fetchXSyndicationProfile(screenName, sourceUrl, timeoutMs) {
|
|
94
|
+
if (!allowXSyndication())
|
|
95
|
+
return null;
|
|
96
|
+
return fetchProviderText(`https://syndication.twitter.com/srv/timeline-profile/screen-name/${encodeURIComponent(screenName)}`, timeoutMs, {
|
|
97
|
+
headers: {
|
|
98
|
+
accept: "text/html,application/xhtml+xml,*/*",
|
|
99
|
+
"user-agent": USER_AGENT,
|
|
100
|
+
},
|
|
101
|
+
}, {
|
|
102
|
+
mode: "relay",
|
|
103
|
+
sourceUrl,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
function asRecord(value) {
|
|
107
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
108
|
+
}
|
|
109
|
+
function asRecordArray(value) {
|
|
110
|
+
return Array.isArray(value) ? value.map(asRecord).filter((item) => Object.keys(item).length > 0) : [];
|
|
111
|
+
}
|
|
59
112
|
function collectFxMediaLinks(tweet) {
|
|
60
113
|
const media = Array.isArray(tweet?.media?.all)
|
|
61
114
|
? tweet.media.all
|
|
@@ -70,6 +123,57 @@ function collectFxMediaLinks(tweet) {
|
|
|
70
123
|
.filter(Boolean)
|
|
71
124
|
.slice(0, 20);
|
|
72
125
|
}
|
|
126
|
+
function collectSyndicationMediaLinks(tweet) {
|
|
127
|
+
const entities = asRecord(tweet.entities);
|
|
128
|
+
const media = asRecordArray(entities.media);
|
|
129
|
+
return media.map((item) => asString(item.media_url_https || item.media_url).trim()).filter(Boolean).slice(0, 20);
|
|
130
|
+
}
|
|
131
|
+
function parseSyndicationTimeline(html) {
|
|
132
|
+
const match = html.match(/<script[^>]*id=["']__NEXT_DATA__["'][^>]*>([\s\S]*?)<\/script>/i)
|
|
133
|
+
|| html.match(/__NEXT_DATA__[\s\S]*?>([\s\S]*?)<\/script>/i);
|
|
134
|
+
if (!match?.[1])
|
|
135
|
+
return [];
|
|
136
|
+
try {
|
|
137
|
+
const data = asRecord(JSON.parse(decodeHtml(match[1])));
|
|
138
|
+
const props = asRecord(data.props);
|
|
139
|
+
const pageProps = asRecord(props.pageProps);
|
|
140
|
+
const timeline = asRecord(pageProps.timeline);
|
|
141
|
+
const entries = asRecordArray(timeline.entries);
|
|
142
|
+
const tweets = [];
|
|
143
|
+
for (const entry of entries) {
|
|
144
|
+
if (asString(entry.type) !== "tweet")
|
|
145
|
+
continue;
|
|
146
|
+
const content = asRecord(entry.content);
|
|
147
|
+
const tweet = asRecord(content.tweet);
|
|
148
|
+
if (Object.keys(tweet).length > 0)
|
|
149
|
+
tweets.push(tweet);
|
|
150
|
+
}
|
|
151
|
+
return tweets;
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
return [];
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
function timelineItemFromTweet(tweet) {
|
|
158
|
+
const user = asRecord(tweet.user);
|
|
159
|
+
const screenName = asString(user.screen_name);
|
|
160
|
+
const id = asString(tweet.id_str || tweet.id);
|
|
161
|
+
const text = decodeHtml(asString(tweet.full_text || tweet.text)).trim();
|
|
162
|
+
const media = collectSyndicationMediaLinks(tweet);
|
|
163
|
+
return {
|
|
164
|
+
title: screenName ? `@${screenName}` : "X post",
|
|
165
|
+
url: screenName && id ? `https://x.com/${screenName}/status/${id}` : "",
|
|
166
|
+
snippet: text,
|
|
167
|
+
text,
|
|
168
|
+
created_at: asString(tweet.created_at),
|
|
169
|
+
author_name: asString(user.name),
|
|
170
|
+
author_screen_name: screenName,
|
|
171
|
+
likes: asNumber(tweet.favorite_count),
|
|
172
|
+
retweets: asNumber(tweet.retweet_count),
|
|
173
|
+
replies: asNumber(tweet.reply_count),
|
|
174
|
+
media,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
73
177
|
function extractArticleBlocksText(article) {
|
|
74
178
|
const blocks = article?.content?.blocks;
|
|
75
179
|
if (!Array.isArray(blocks) || blocks.length === 0)
|
|
@@ -115,10 +219,78 @@ function textFromTweet(tweet) {
|
|
|
115
219
|
function titleFromTweet(tweet) {
|
|
116
220
|
return asString(tweet?.article?.title || tweet?.twitter_card?.title || textFromTweet(tweet).slice(0, 140) || `Post by @${asString(tweet?.author?.screen_name || "x")}`);
|
|
117
221
|
}
|
|
222
|
+
function extractOembedCreatedAt(html) {
|
|
223
|
+
const match = html.match(/—[\s\S]*?<a\b[^>]*>([\s\S]*?)<\/a>/i);
|
|
224
|
+
return match?.[1] ? stripTags(match[1]).trim() : "";
|
|
225
|
+
}
|
|
226
|
+
function extractOembedScreenName(html, authorUrl) {
|
|
227
|
+
const fromUrl = extractXScreenNameFromUrl(authorUrl);
|
|
228
|
+
if (fromUrl)
|
|
229
|
+
return fromUrl;
|
|
230
|
+
const match = stripTags(html).match(/@([A-Za-z0-9_]{1,15})\b/);
|
|
231
|
+
return match?.[1] || "";
|
|
232
|
+
}
|
|
233
|
+
function profileUnavailable(screenName, context, errorCode, message) {
|
|
234
|
+
return {
|
|
235
|
+
post: {
|
|
236
|
+
url: context.url,
|
|
237
|
+
title: `X profile timeline: @${screenName}`,
|
|
238
|
+
text: `X profile timeline for @${screenName} is unavailable via public syndication. Fetch a specific post URL, or discover posts with smartsearch query "site:x.com/${screenName}". Private, deleted, rate-limited, or authwalled profiles require browser/user handoff.`,
|
|
239
|
+
status: "blocked_or_unavailable",
|
|
240
|
+
kind: "x_profile",
|
|
241
|
+
author_screen_name: screenName,
|
|
242
|
+
},
|
|
243
|
+
thread: [],
|
|
244
|
+
comments: [],
|
|
245
|
+
outbound_links: [],
|
|
246
|
+
partial: true,
|
|
247
|
+
errors: [
|
|
248
|
+
...context.errors,
|
|
249
|
+
{ category: "unavailable", code: errorCode, message },
|
|
250
|
+
],
|
|
251
|
+
method: context.active.method,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
118
254
|
export const xProvider = {
|
|
119
255
|
id: "x",
|
|
120
256
|
matches: (_url, target) => target === "x_post",
|
|
121
257
|
async normalize(context) {
|
|
258
|
+
const profile = extractXProfile(context.url);
|
|
259
|
+
if (profile) {
|
|
260
|
+
const timeline = await fetchXSyndicationProfile(profile.screenName, context.url, context.timeoutMs);
|
|
261
|
+
if (timeline?.ok) {
|
|
262
|
+
const items = parseSyndicationTimeline(timeline.text).map(timelineItemFromTweet).filter((item) => item.text || item.url).slice(0, 20);
|
|
263
|
+
if (items.length > 0) {
|
|
264
|
+
const first = items[0];
|
|
265
|
+
const media = items.flatMap((item) => item.media).slice(0, 20);
|
|
266
|
+
const displayName = asString(first.author_name || profile.screenName);
|
|
267
|
+
return {
|
|
268
|
+
post: {
|
|
269
|
+
url: context.url,
|
|
270
|
+
title: `X profile timeline: ${displayName} (@${profile.screenName})`,
|
|
271
|
+
text: `Recent public X posts for @${profile.screenName}.`,
|
|
272
|
+
status: "ok",
|
|
273
|
+
kind: "x_profile",
|
|
274
|
+
author_name: displayName,
|
|
275
|
+
author_screen_name: profile.screenName,
|
|
276
|
+
media,
|
|
277
|
+
},
|
|
278
|
+
thread: items,
|
|
279
|
+
comments: [],
|
|
280
|
+
outbound_links: dedupeUrls([
|
|
281
|
+
...items.map((item) => item.url),
|
|
282
|
+
...media,
|
|
283
|
+
]),
|
|
284
|
+
partial: false,
|
|
285
|
+
errors: context.errors.filter((item) => item.code !== "playwright_fetch_failed" && !item.code.startsWith("uc_")),
|
|
286
|
+
method: "x_syndication_profile",
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
return profileUnavailable(profile.screenName, context, "x_profile_empty", "X public syndication returned no profile timeline posts");
|
|
290
|
+
}
|
|
291
|
+
const error = timeline?.error;
|
|
292
|
+
return profileUnavailable(profile.screenName, context, error?.code || "x_profile_unavailable", error?.message || "X public syndication profile timeline is unavailable");
|
|
293
|
+
}
|
|
122
294
|
const fx = await fetchFxTwitterStatus(context.url, context.timeoutMs);
|
|
123
295
|
if (fx?.tweet) {
|
|
124
296
|
const tweet = fx.tweet || {};
|
|
@@ -203,15 +375,30 @@ export const xProvider = {
|
|
|
203
375
|
mode: "relay",
|
|
204
376
|
sourceUrl: context.url,
|
|
205
377
|
});
|
|
206
|
-
const
|
|
378
|
+
const fallbackData = asRecord(fallback.ok ? fallback.data : null);
|
|
379
|
+
const html = fallback.ok ? asString(fallbackData.html) : "";
|
|
207
380
|
const match = html.match(/<p[^>]*>([\s\S]*?)<\/p>/i);
|
|
208
381
|
const oembedText = match?.[1] ? stripTags(match[1]) : "";
|
|
382
|
+
const authorName = asString(fallbackData.author_name);
|
|
383
|
+
const authorUrl = asString(fallbackData.author_url);
|
|
384
|
+
const canonicalUrl = asString(fallbackData.url || context.url);
|
|
385
|
+
const screenName = extractOembedScreenName(html, authorUrl);
|
|
386
|
+
const createdAt = extractOembedCreatedAt(html);
|
|
209
387
|
const providerError = !fallback.ok ? fallback.error : null;
|
|
210
388
|
return {
|
|
211
|
-
post: {
|
|
389
|
+
post: {
|
|
390
|
+
url: canonicalUrl || context.url,
|
|
391
|
+
title: oembedText ? `Post by ${authorName || (screenName ? `@${screenName}` : "X")}${authorName && screenName ? ` (@${screenName})` : ""}` : "",
|
|
392
|
+
text: oembedText,
|
|
393
|
+
status: oembedText ? "ok" : "blocked_or_unavailable",
|
|
394
|
+
author_name: authorName,
|
|
395
|
+
author_screen_name: screenName,
|
|
396
|
+
author_url: authorUrl,
|
|
397
|
+
created_at: createdAt,
|
|
398
|
+
},
|
|
212
399
|
thread: [],
|
|
213
400
|
comments: [],
|
|
214
|
-
outbound_links: dedupeUrls([...extractUrls(
|
|
401
|
+
outbound_links: dedupeUrls([canonicalUrl, authorUrl, ...extractUrls(oembedText), ...context.active.links]),
|
|
215
402
|
partial: true,
|
|
216
403
|
errors: oembedText
|
|
217
404
|
? context.errors.filter((item) => item.code !== "playwright_fetch_failed" && !item.code.startsWith("uc_"))
|