hermes-plugin-tinyfish 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,11 @@
1
+ .DS_Store
2
+ .coverage
3
+ .mypy_cache/
4
+ .pytest_cache/
5
+ .ruff_cache/
6
+ .venv/
7
+ __pycache__/
8
+ build/
9
+ dist/
10
+ *.egg-info/
11
+ htmlcov/
@@ -0,0 +1,35 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project uses semantic versioning.
7
+
8
+ ## [0.1.3] - 2026-07-07
9
+
10
+ ### Changed
11
+
12
+ - Enabled PyPI publication through Trusted Publishing.
13
+
14
+ ## [0.1.2] - 2026-07-06
15
+
16
+ ### Changed
17
+
18
+ - Corrected public package author metadata to use the repository owner handle.
19
+
20
+ ## [0.1.1] - 2026-07-06
21
+
22
+ ### Changed
23
+
24
+ - Clarified that Hermes Git plugin installs clone the repository default branch.
25
+ - Added gated PyPI publishing directly to the tag-based release workflow for future releases.
26
+
27
+ ## [0.1.0] - 2026-07-06
28
+
29
+ ### Added
30
+
31
+ - Initial TinyFish Hermes web provider.
32
+ - OAuth MCP-first routing for search and fetch.
33
+ - REST API-key fallback for `TINYFISH_API_KEY`.
34
+ - `hermes tinyfish setup`, `doctor`, and `status` CLI commands.
35
+ - CI, release, dependency review, and PyPI publishing workflows.
@@ -0,0 +1,9 @@
1
+ # Code of Conduct
2
+
3
+ This project follows the Contributor Covenant Code of Conduct.
4
+
5
+ Be respectful, constructive, and specific. Harassment, abuse, and bad-faith
6
+ participation are not acceptable in issues, pull requests, or discussions.
7
+
8
+ Maintainers may remove comments, close threads, or block users who violate
9
+ these expectations.
@@ -0,0 +1,25 @@
1
+ # Contributing
2
+
3
+ Contributions are welcome.
4
+
5
+ ## Development
6
+
7
+ ```bash
8
+ python -m venv .venv
9
+ source .venv/bin/activate
10
+ python -m pip install -U pip
11
+ python -m pip install -e ".[dev]"
12
+ ruff format .
13
+ ruff check .
14
+ mypy hermes_plugin_tinyfish
15
+ pytest
16
+ python -m build
17
+ ```
18
+
19
+ ## Pull Request Expectations
20
+
21
+ - Include tests for behavior changes.
22
+ - Keep the plugin upgrade-safe: do not modify Hermes core files, local update
23
+ scripts, Dockerfiles, or generated user state.
24
+ - Do not log or print API keys, OAuth tokens, or MCP token file contents.
25
+ - Use TinyFish MCP OAuth first when possible and REST API keys only as fallback.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 gabeosx
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,165 @@
1
+ Metadata-Version: 2.4
2
+ Name: hermes-plugin-tinyfish
3
+ Version: 0.1.3
4
+ Summary: TinyFish web search and extraction provider plugin for Hermes Agent
5
+ Project-URL: Homepage, https://github.com/gabeosx/hermes-plugin-tinyfish
6
+ Project-URL: Documentation, https://github.com/gabeosx/hermes-plugin-tinyfish#readme
7
+ Project-URL: Issues, https://github.com/gabeosx/hermes-plugin-tinyfish/issues
8
+ Project-URL: Changelog, https://github.com/gabeosx/hermes-plugin-tinyfish/blob/main/CHANGELOG.md
9
+ Author: gabeosx
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: hermes-agent,mcp,tinyfish,web-extraction,web-search
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
21
+ Requires-Python: >=3.10
22
+ Requires-Dist: httpx>=0.27
23
+ Provides-Extra: dev
24
+ Requires-Dist: build>=1.2; extra == 'dev'
25
+ Requires-Dist: mypy>=1.10; extra == 'dev'
26
+ Requires-Dist: pytest-cov>=5.0; extra == 'dev'
27
+ Requires-Dist: pytest>=8.2; extra == 'dev'
28
+ Requires-Dist: ruff>=0.6; extra == 'dev'
29
+ Description-Content-Type: text/markdown
30
+
31
+ # Hermes TinyFish Plugin
32
+
33
+ [![CI](https://github.com/gabeosx/hermes-plugin-tinyfish/actions/workflows/ci.yml/badge.svg)](https://github.com/gabeosx/hermes-plugin-tinyfish/actions/workflows/ci.yml)
34
+ [![PyPI](https://img.shields.io/badge/PyPI-pending-lightgrey.svg)](https://github.com/gabeosx/hermes-plugin-tinyfish/releases)
35
+ [![Python](https://img.shields.io/badge/Python-3.10%20%7C%203.11%20%7C%203.12-blue.svg)](pyproject.toml)
36
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
37
+ [![Security Policy](https://img.shields.io/badge/Security-policy-green.svg)](SECURITY.md)
38
+
39
+ TinyFish web search and content extraction for [Hermes Agent](https://hermes-agent.nousresearch.com/docs/).
40
+
41
+ The plugin adds `tinyfish` as a native Hermes `web_search` and `web_extract`
42
+ backend. It prefers TinyFish's hosted OAuth MCP server and falls back to direct
43
+ REST API calls with `TINYFISH_API_KEY` when MCP OAuth is unavailable.
44
+
45
+ ## Install
46
+
47
+ Hermes Git plugin install:
48
+
49
+ ```bash
50
+ hermes plugins install gabeosx/hermes-plugin-tinyfish --enable
51
+ hermes tinyfish setup
52
+ ```
53
+
54
+ Hermes' Git plugin installer currently clones the repository default branch.
55
+ Stable release artifacts are published on the
56
+ [GitHub Releases page](https://github.com/gabeosx/hermes-plugin-tinyfish/releases).
57
+
58
+ PyPI install is planned after PyPI Trusted Publishing is configured:
59
+
60
+ ```bash
61
+ pip install hermes-plugin-tinyfish
62
+ hermes plugins enable web-tinyfish
63
+ hermes tinyfish setup
64
+ ```
65
+
66
+ ## What Setup Does
67
+
68
+ `hermes tinyfish setup` writes ordinary user configuration only:
69
+
70
+ ```yaml
71
+ mcp_servers:
72
+ tinyfish:
73
+ url: https://agent.tinyfish.ai/mcp
74
+ auth: oauth
75
+ tools:
76
+ include: [search, fetch_content]
77
+ resources: false
78
+ prompts: false
79
+
80
+ web:
81
+ search_backend: tinyfish
82
+ extract_backend: tinyfish
83
+ ```
84
+
85
+ It may also save `TINYFISH_API_KEY` to `~/.hermes/.env` if you choose to add
86
+ an API-key fallback.
87
+
88
+ The plugin does not patch Hermes Agent, update scripts, Dockerfiles, or any
89
+ files inside a Hermes checkout.
90
+
91
+ ## Verify
92
+
93
+ ```bash
94
+ hermes tinyfish doctor
95
+ hermes tinyfish doctor --live
96
+ ```
97
+
98
+ `doctor --live` performs a real TinyFish search and fetch. It requires either
99
+ working MCP OAuth or `TINYFISH_API_KEY`.
100
+
101
+ ## Manual Configuration
102
+
103
+ OAuth MCP only:
104
+
105
+ ```bash
106
+ hermes mcp add tinyfish --url https://agent.tinyfish.ai/mcp --auth oauth
107
+ hermes mcp login tinyfish
108
+ hermes mcp configure tinyfish
109
+ ```
110
+
111
+ Then set:
112
+
113
+ ```yaml
114
+ web:
115
+ search_backend: tinyfish
116
+ extract_backend: tinyfish
117
+ ```
118
+
119
+ REST fallback only:
120
+
121
+ ```bash
122
+ export TINYFISH_API_KEY="..."
123
+ ```
124
+
125
+ ## Upgrade Safety
126
+
127
+ This plugin uses public Hermes extension points:
128
+
129
+ - `ctx.register_web_search_provider(...)`
130
+ - `ctx.register_cli_command(...)`
131
+ - Hermes MCP config under `mcp_servers`
132
+ - Hermes `.env` config helpers for optional API-key fallback
133
+
134
+ Because it is installed as a user plugin or Python entry point, normal
135
+ `hermes update` operations do not overwrite it.
136
+
137
+ ## Development
138
+
139
+ ```bash
140
+ python -m venv .venv
141
+ source .venv/bin/activate
142
+ python -m pip install -U pip
143
+ python -m pip install -e ".[dev]"
144
+ ruff format .
145
+ ruff check .
146
+ mypy hermes_plugin_tinyfish
147
+ pytest
148
+ python -m build
149
+ ```
150
+
151
+ Live tests are opt-in:
152
+
153
+ ```bash
154
+ TINYFISH_LIVE_TESTS=1 TINYFISH_API_KEY=... pytest tests/test_live.py
155
+ ```
156
+
157
+ ## References
158
+
159
+ - [TinyFish MCP Integration](https://docs.tinyfish.ai/mcp-integration)
160
+ - [TinyFish Authentication](https://docs.tinyfish.ai/authentication)
161
+ - [TinyFish Search API](https://docs.tinyfish.ai/search-api)
162
+ - [TinyFish Fetch API](https://docs.tinyfish.ai/fetch-api)
163
+ - [Hermes Web Search Provider Plugins](https://hermes-agent.nousresearch.com/docs/developer-guide/web-search-provider-plugin)
164
+ - [Hermes Plugins](https://hermes-agent.nousresearch.com/docs/user-guide/features/plugins)
165
+ - [Hermes MCP](https://hermes-agent.nousresearch.com/docs/user-guide/features/mcp)
@@ -0,0 +1,135 @@
1
+ # Hermes TinyFish Plugin
2
+
3
+ [![CI](https://github.com/gabeosx/hermes-plugin-tinyfish/actions/workflows/ci.yml/badge.svg)](https://github.com/gabeosx/hermes-plugin-tinyfish/actions/workflows/ci.yml)
4
+ [![PyPI](https://img.shields.io/badge/PyPI-pending-lightgrey.svg)](https://github.com/gabeosx/hermes-plugin-tinyfish/releases)
5
+ [![Python](https://img.shields.io/badge/Python-3.10%20%7C%203.11%20%7C%203.12-blue.svg)](pyproject.toml)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
7
+ [![Security Policy](https://img.shields.io/badge/Security-policy-green.svg)](SECURITY.md)
8
+
9
+ TinyFish web search and content extraction for [Hermes Agent](https://hermes-agent.nousresearch.com/docs/).
10
+
11
+ The plugin adds `tinyfish` as a native Hermes `web_search` and `web_extract`
12
+ backend. It prefers TinyFish's hosted OAuth MCP server and falls back to direct
13
+ REST API calls with `TINYFISH_API_KEY` when MCP OAuth is unavailable.
14
+
15
+ ## Install
16
+
17
+ Hermes Git plugin install:
18
+
19
+ ```bash
20
+ hermes plugins install gabeosx/hermes-plugin-tinyfish --enable
21
+ hermes tinyfish setup
22
+ ```
23
+
24
+ Hermes' Git plugin installer currently clones the repository default branch.
25
+ Stable release artifacts are published on the
26
+ [GitHub Releases page](https://github.com/gabeosx/hermes-plugin-tinyfish/releases).
27
+
28
+ PyPI install is planned after PyPI Trusted Publishing is configured:
29
+
30
+ ```bash
31
+ pip install hermes-plugin-tinyfish
32
+ hermes plugins enable web-tinyfish
33
+ hermes tinyfish setup
34
+ ```
35
+
36
+ ## What Setup Does
37
+
38
+ `hermes tinyfish setup` writes ordinary user configuration only:
39
+
40
+ ```yaml
41
+ mcp_servers:
42
+ tinyfish:
43
+ url: https://agent.tinyfish.ai/mcp
44
+ auth: oauth
45
+ tools:
46
+ include: [search, fetch_content]
47
+ resources: false
48
+ prompts: false
49
+
50
+ web:
51
+ search_backend: tinyfish
52
+ extract_backend: tinyfish
53
+ ```
54
+
55
+ It may also save `TINYFISH_API_KEY` to `~/.hermes/.env` if you choose to add
56
+ an API-key fallback.
57
+
58
+ The plugin does not patch Hermes Agent, update scripts, Dockerfiles, or any
59
+ files inside a Hermes checkout.
60
+
61
+ ## Verify
62
+
63
+ ```bash
64
+ hermes tinyfish doctor
65
+ hermes tinyfish doctor --live
66
+ ```
67
+
68
+ `doctor --live` performs a real TinyFish search and fetch. It requires either
69
+ working MCP OAuth or `TINYFISH_API_KEY`.
70
+
71
+ ## Manual Configuration
72
+
73
+ OAuth MCP only:
74
+
75
+ ```bash
76
+ hermes mcp add tinyfish --url https://agent.tinyfish.ai/mcp --auth oauth
77
+ hermes mcp login tinyfish
78
+ hermes mcp configure tinyfish
79
+ ```
80
+
81
+ Then set:
82
+
83
+ ```yaml
84
+ web:
85
+ search_backend: tinyfish
86
+ extract_backend: tinyfish
87
+ ```
88
+
89
+ REST fallback only:
90
+
91
+ ```bash
92
+ export TINYFISH_API_KEY="..."
93
+ ```
94
+
95
+ ## Upgrade Safety
96
+
97
+ This plugin uses public Hermes extension points:
98
+
99
+ - `ctx.register_web_search_provider(...)`
100
+ - `ctx.register_cli_command(...)`
101
+ - Hermes MCP config under `mcp_servers`
102
+ - Hermes `.env` config helpers for optional API-key fallback
103
+
104
+ Because it is installed as a user plugin or Python entry point, normal
105
+ `hermes update` operations do not overwrite it.
106
+
107
+ ## Development
108
+
109
+ ```bash
110
+ python -m venv .venv
111
+ source .venv/bin/activate
112
+ python -m pip install -U pip
113
+ python -m pip install -e ".[dev]"
114
+ ruff format .
115
+ ruff check .
116
+ mypy hermes_plugin_tinyfish
117
+ pytest
118
+ python -m build
119
+ ```
120
+
121
+ Live tests are opt-in:
122
+
123
+ ```bash
124
+ TINYFISH_LIVE_TESTS=1 TINYFISH_API_KEY=... pytest tests/test_live.py
125
+ ```
126
+
127
+ ## References
128
+
129
+ - [TinyFish MCP Integration](https://docs.tinyfish.ai/mcp-integration)
130
+ - [TinyFish Authentication](https://docs.tinyfish.ai/authentication)
131
+ - [TinyFish Search API](https://docs.tinyfish.ai/search-api)
132
+ - [TinyFish Fetch API](https://docs.tinyfish.ai/fetch-api)
133
+ - [Hermes Web Search Provider Plugins](https://hermes-agent.nousresearch.com/docs/developer-guide/web-search-provider-plugin)
134
+ - [Hermes Plugins](https://hermes-agent.nousresearch.com/docs/user-guide/features/plugins)
135
+ - [Hermes MCP](https://hermes-agent.nousresearch.com/docs/user-guide/features/mcp)
@@ -0,0 +1,20 @@
1
+ # Security Policy
2
+
3
+ ## Supported Versions
4
+
5
+ Security fixes are released for the latest minor version.
6
+
7
+ ## Reporting a Vulnerability
8
+
9
+ Please report security issues privately by opening a GitHub security advisory
10
+ for this repository. Do not disclose vulnerabilities in public issues until a
11
+ fix is available.
12
+
13
+ ## Design Notes
14
+
15
+ - The plugin does not modify Hermes Agent source files.
16
+ - The preferred TinyFish authentication path is Hermes' OAuth MCP support.
17
+ - API keys are only used as an explicit REST fallback through
18
+ `TINYFISH_API_KEY`.
19
+ - `hermes tinyfish status` and `doctor` never print secret values.
20
+ - Live tests are opt-in and are not run by default in CI.
@@ -0,0 +1,15 @@
1
+ """Directory-plugin entry point for ``hermes plugins install``.
2
+
3
+ Hermes loads this file directly when the repository is installed as a user
4
+ plugin. The package entry point in ``pyproject.toml`` imports the same
5
+ ``register`` function when installed from PyPI.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ try:
11
+ from .hermes_plugin_tinyfish import register
12
+ except ImportError: # pragma: no cover - used for pip entry-point import shape
13
+ from hermes_plugin_tinyfish import register
14
+
15
+ __all__ = ["register"]
@@ -0,0 +1,35 @@
1
+ """TinyFish provider plugin for Hermes Agent."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ __version__ = "0.1.0"
8
+
9
+
10
+ def register(ctx: Any) -> None:
11
+ """Register the TinyFish web provider and CLI setup commands."""
12
+
13
+ from .provider import TinyFishWebSearchProvider
14
+ from .setup_cli import dispatch_tinyfish_cli, setup_tinyfish_cli
15
+
16
+ provider = TinyFishWebSearchProvider(dispatch_tool=getattr(ctx, "dispatch_tool", None))
17
+ ctx.register_web_search_provider(provider)
18
+ ctx.register_cli_command(
19
+ name="tinyfish",
20
+ help="Configure and diagnose the TinyFish web provider",
21
+ setup_fn=setup_tinyfish_cli,
22
+ handler_fn=dispatch_tinyfish_cli,
23
+ description="Configure TinyFish MCP OAuth, API-key fallback, and Hermes web backend routing.",
24
+ )
25
+
26
+
27
+ def __getattr__(name: str) -> Any:
28
+ if name == "TinyFishWebSearchProvider":
29
+ from .provider import TinyFishWebSearchProvider
30
+
31
+ return TinyFishWebSearchProvider
32
+ raise AttributeError(name)
33
+
34
+
35
+ __all__ = ["TinyFishWebSearchProvider", "register"]
@@ -0,0 +1,175 @@
1
+ """Normalize TinyFish MCP and REST payloads into Hermes web-provider shapes."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from typing import Any, cast
7
+
8
+
9
+ class TinyFishPayloadError(ValueError):
10
+ """Raised when a TinyFish or MCP payload reports an error."""
11
+
12
+
13
+ def parse_jsonish(value: Any) -> Any:
14
+ """Parse JSON strings when possible and return other values unchanged."""
15
+
16
+ if isinstance(value, str):
17
+ text = value.strip()
18
+ if not text:
19
+ return text
20
+ try:
21
+ return json.loads(text)
22
+ except json.JSONDecodeError:
23
+ return value
24
+ return value
25
+
26
+
27
+ def unwrap_mcp_payload(raw: Any) -> Any:
28
+ """Convert Hermes' MCP wrapper output to the underlying TinyFish payload."""
29
+
30
+ payload = parse_jsonish(raw)
31
+ if isinstance(payload, dict) and payload.get("error"):
32
+ raise TinyFishPayloadError(str(payload["error"]))
33
+
34
+ if isinstance(payload, dict) and "structuredContent" in payload:
35
+ structured = payload.get("structuredContent")
36
+ if structured is not None:
37
+ return structured
38
+
39
+ if isinstance(payload, dict) and "result" in payload:
40
+ result = parse_jsonish(payload.get("result"))
41
+ if isinstance(result, dict) and result.get("error"):
42
+ raise TinyFishPayloadError(str(result["error"]))
43
+ return result
44
+
45
+ return payload
46
+
47
+
48
+ def normalize_search_response(payload: Any, limit: int = 5) -> dict[str, Any]:
49
+ """Return Hermes' standard web-search response envelope."""
50
+
51
+ data = unwrap_mcp_payload(payload)
52
+ if isinstance(data, list):
53
+ results = data
54
+ elif isinstance(data, dict):
55
+ if data.get("error"):
56
+ raise TinyFishPayloadError(str(data["error"]))
57
+ if isinstance(data.get("data"), dict) and isinstance(data["data"].get("web"), list):
58
+ return {
59
+ "success": True,
60
+ "data": {"web": data["data"]["web"][: max(1, int(limit or 5))]},
61
+ }
62
+ results = data.get("results") or data.get("web") or []
63
+ else:
64
+ results = []
65
+
66
+ count = max(1, int(limit or 5))
67
+ web_results: list[dict[str, Any]] = []
68
+ for idx, item in enumerate(list(results)[:count]):
69
+ if not isinstance(item, dict):
70
+ continue
71
+ url = str(item.get("url") or item.get("link") or "")
72
+ web_results.append(
73
+ {
74
+ "title": str(item.get("title") or item.get("site_name") or url),
75
+ "url": url,
76
+ "description": str(
77
+ item.get("snippet")
78
+ or item.get("description")
79
+ or item.get("content")
80
+ or item.get("text")
81
+ or ""
82
+ ),
83
+ "position": int(item.get("position") or idx + 1),
84
+ }
85
+ )
86
+ return {"success": True, "data": {"web": web_results}}
87
+
88
+
89
+ def normalize_fetch_documents(payload: Any, fallback_urls: list[str] | None = None) -> list[dict[str, Any]]:
90
+ """Return Hermes' standard extract document list."""
91
+
92
+ urls = list(fallback_urls or [])
93
+ data = unwrap_mcp_payload(payload)
94
+ if isinstance(data, dict) and data.get("error"):
95
+ raise TinyFishPayloadError(str(data["error"]))
96
+
97
+ if isinstance(data, dict) and isinstance(data.get("data"), list):
98
+ return cast(list[dict[str, Any]], data["data"])
99
+
100
+ if isinstance(data, dict):
101
+ results = data.get("results") or data.get("documents") or []
102
+ errors = data.get("errors") or data.get("failed_results") or []
103
+ elif isinstance(data, list):
104
+ results = data
105
+ errors = []
106
+ else:
107
+ results = []
108
+ errors = []
109
+
110
+ documents: list[dict[str, Any]] = []
111
+ for idx, item in enumerate(list(results)):
112
+ if isinstance(item, str):
113
+ url = urls[idx] if idx < len(urls) else ""
114
+ documents.append(
115
+ {
116
+ "url": url,
117
+ "title": "",
118
+ "content": item,
119
+ "raw_content": item,
120
+ "metadata": {"sourceURL": url},
121
+ }
122
+ )
123
+ continue
124
+ if not isinstance(item, dict):
125
+ continue
126
+ url = str(item.get("url") or item.get("final_url") or (urls[idx] if idx < len(urls) else ""))
127
+ raw = str(
128
+ item.get("text") or item.get("markdown") or item.get("raw_content") or item.get("content") or ""
129
+ )
130
+ documents.append(
131
+ {
132
+ "url": url,
133
+ "title": str(item.get("title") or ""),
134
+ "content": raw,
135
+ "raw_content": raw,
136
+ "metadata": {
137
+ "sourceURL": url,
138
+ "finalURL": str(item.get("final_url") or url),
139
+ "description": str(item.get("description") or ""),
140
+ "language": str(item.get("language") or ""),
141
+ },
142
+ }
143
+ )
144
+
145
+ for idx, item in enumerate(list(errors)):
146
+ if isinstance(item, dict):
147
+ url = str(item.get("url") or (urls[idx] if idx < len(urls) else ""))
148
+ error = str(item.get("error") or item.get("message") or "fetch failed")
149
+ else:
150
+ url = urls[idx] if idx < len(urls) else ""
151
+ error = str(item)
152
+ documents.append(
153
+ {
154
+ "url": url,
155
+ "title": "",
156
+ "content": "",
157
+ "raw_content": "",
158
+ "error": error,
159
+ "metadata": {"sourceURL": url},
160
+ }
161
+ )
162
+
163
+ if not documents and urls:
164
+ return [
165
+ {
166
+ "url": url,
167
+ "title": "",
168
+ "content": "",
169
+ "raw_content": "",
170
+ "error": "TinyFish returned no content",
171
+ "metadata": {"sourceURL": url},
172
+ }
173
+ for url in urls
174
+ ]
175
+ return documents