omnireach 0.11.0a0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omnireach-0.11.0a0/.gitignore +19 -0
- omnireach-0.11.0a0/LICENSE +21 -0
- omnireach-0.11.0a0/PKG-INFO +387 -0
- omnireach-0.11.0a0/README.md +354 -0
- omnireach-0.11.0a0/README.zh.md +354 -0
- omnireach-0.11.0a0/omnireach/__init__.py +3 -0
- omnireach-0.11.0a0/omnireach/__main__.py +4 -0
- omnireach-0.11.0a0/omnireach/adapters/__init__.py +0 -0
- omnireach-0.11.0a0/omnireach/adapters/_bilibili_api.py +104 -0
- omnireach-0.11.0a0/omnireach/adapters/_wechat_sogou.py +134 -0
- omnireach-0.11.0a0/omnireach/adapters/base.py +43 -0
- omnireach-0.11.0a0/omnireach/adapters/bilibili.py +72 -0
- omnireach-0.11.0a0/omnireach/adapters/brave.py +52 -0
- omnireach-0.11.0a0/omnireach/adapters/douyin.py +76 -0
- omnireach-0.11.0a0/omnireach/adapters/exa.py +59 -0
- omnireach-0.11.0a0/omnireach/adapters/github.py +68 -0
- omnireach-0.11.0a0/omnireach/adapters/hackernews.py +52 -0
- omnireach-0.11.0a0/omnireach/adapters/perplexity.py +59 -0
- omnireach-0.11.0a0/omnireach/adapters/reddit.py +67 -0
- omnireach-0.11.0a0/omnireach/adapters/rss.py +56 -0
- omnireach-0.11.0a0/omnireach/adapters/tavily.py +53 -0
- omnireach-0.11.0a0/omnireach/adapters/tiktok.py +75 -0
- omnireach-0.11.0a0/omnireach/adapters/twitter.py +73 -0
- omnireach-0.11.0a0/omnireach/adapters/web.py +54 -0
- omnireach-0.11.0a0/omnireach/adapters/wechat.py +72 -0
- omnireach-0.11.0a0/omnireach/adapters/xiaohongshu.py +80 -0
- omnireach-0.11.0a0/omnireach/adapters/youtube.py +65 -0
- omnireach-0.11.0a0/omnireach/cli.py +257 -0
- omnireach-0.11.0a0/omnireach/commands/__init__.py +0 -0
- omnireach-0.11.0a0/omnireach/commands/check_update.py +49 -0
- omnireach-0.11.0a0/omnireach/commands/fetch.py +267 -0
- omnireach-0.11.0a0/omnireach/commands/init.py +43 -0
- omnireach-0.11.0a0/omnireach/commands/preferences.py +74 -0
- omnireach-0.11.0a0/omnireach/commands/setup.py +252 -0
- omnireach-0.11.0a0/omnireach/commands/sources.py +118 -0
- omnireach-0.11.0a0/omnireach/contract.py +65 -0
- omnireach-0.11.0a0/omnireach/dispatcher.py +61 -0
- omnireach-0.11.0a0/omnireach/doctor.py +221 -0
- omnireach-0.11.0a0/omnireach/installer.py +44 -0
- omnireach-0.11.0a0/omnireach/normalizer.py +18 -0
- omnireach-0.11.0a0/omnireach/preferences.py +84 -0
- omnireach-0.11.0a0/omnireach/registry.py +88 -0
- omnireach-0.11.0a0/omnireach/router.py +62 -0
- omnireach-0.11.0a0/omnireach/scorer.py +46 -0
- omnireach-0.11.0a0/omnireach/secrets_env.py +47 -0
- omnireach-0.11.0a0/omnireach/sources.yml +211 -0
- omnireach-0.11.0a0/omnireach/wizard.py +133 -0
- omnireach-0.11.0a0/pyproject.toml +60 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
*.egg-info/
|
|
4
|
+
.venv/
|
|
5
|
+
venv/
|
|
6
|
+
dist/
|
|
7
|
+
build/
|
|
8
|
+
.pytest_cache/
|
|
9
|
+
.coverage
|
|
10
|
+
.DS_Store
|
|
11
|
+
*.swp
|
|
12
|
+
.claude/
|
|
13
|
+
.env
|
|
14
|
+
|
|
15
|
+
# hyperframes skills + scratch (installed via `npx skills add` to author the demo GIF — not part of the project)
|
|
16
|
+
.agents/
|
|
17
|
+
skills-lock.json
|
|
18
|
+
demo-video/
|
|
19
|
+
docs/launch-drafts/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Daily-AC
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: omnireach
|
|
3
|
+
Version: 0.11.0a0
|
|
4
|
+
Summary: Search & read the login-walled Chinese internet for your AI agent — 微信公众号/小红书/抖音/B站 + Twitter/Reddit/HN/YouTube, one CLI, one JSON schema. Zero-config WeChat search.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Daily-AC/omnireach
|
|
6
|
+
Project-URL: Repository, https://github.com/Daily-AC/omnireach
|
|
7
|
+
Project-URL: Issues, https://github.com/Daily-AC/omnireach/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/Daily-AC/omnireach/releases
|
|
9
|
+
Author-email: Daily-AC <zoned-group@alum.calarts.edu>
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agent,bilibili,claude,cli,douyin,search,skill,wechat,xiaohongshu
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Requires-Dist: click>=8.1
|
|
20
|
+
Requires-Dist: cssselect>=1.4.0
|
|
21
|
+
Requires-Dist: feedparser<7.0,>=6.0
|
|
22
|
+
Requires-Dist: httpx>=0.27
|
|
23
|
+
Requires-Dist: lxml>=6.1.1
|
|
24
|
+
Requires-Dist: pydantic>=2.5
|
|
25
|
+
Requires-Dist: pyyaml>=6.0
|
|
26
|
+
Requires-Dist: rich>=13.7
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pexpect>=4.9; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
31
|
+
Requires-Dist: respx>=0.21; extra == 'dev'
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# omnireach
|
|
35
|
+
|
|
36
|
+
<sub>English · [中文](./README.zh.md)</sub>
|
|
37
|
+
|
|
38
|
+
**Your agent reads Twitter. It still can't read WeChat. omnireach fixes that.**
|
|
39
|
+
|
|
40
|
+
omnireach searches and reads the login-walled Chinese internet — 微信公众号 · 小红书 · 抖音 · B站 · TikTok — plus Twitter, Reddit, HN, YouTube and more, through one uniform interface: `omnireach search` returns one normalized JSON schema across every source, `omnireach fetch` returns clean markdown for any URL. WeChat search works with **zero config** — no key, no login:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
omnireach search --on wechat "Claude Code 技巧" # works 60 seconds after install
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Installed as a Claude Code skill, so your agent just knows how to use it next session. No LLM inside, no per-call scraping fees — the heavy sources reuse your own browser session.
|
|
47
|
+
|
|
48
|
+

|
|
49
|
+
|
|
50
|
+
[](LICENSE)
|
|
51
|
+
[](https://www.python.org/)
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## Install — just tell your agent
|
|
56
|
+
|
|
57
|
+
> **"Install omnireach"**
|
|
58
|
+
|
|
59
|
+
Your agent runs one command. You copy nothing.
|
|
60
|
+
|
|
61
|
+
<sub>Manual fallback — paste into your terminal if you prefer:</sub>
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
curl -fsSL https://raw.githubusercontent.com/Daily-AC/omnireach/main/install.sh | sh
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
This installs the CLI **and** registers the Claude Code skill (auto-discovered next session). Zero-config sources — HackerNews, RSS, 微信 (Sogou path), B站 — work immediately. Other sources open up after a one-time setup step each.
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## What you get
|
|
72
|
+
|
|
73
|
+
**1. Reach the unreachable.**
|
|
74
|
+
Twitter, Reddit, 小红书, 微信公众号, 抖音, B站, TikTok — login-walled vertical platforms that no agent web search reaches. omnireach reads them through your own logged-in browser session (via OpenCLI Chrome bridge), so the results your agent sees are the same results a human would see when signed in.
|
|
75
|
+
|
|
76
|
+
**2. One uniform contract.**
|
|
77
|
+
`omnireach search` → normalized metadata + URL, same shape across every source. `omnireach fetch` → clean markdown for any URL, with host-aware routing (`mp.weixin.qq.com` goes through your logged-in Chrome session to bypass CAPTCHA walls; all other hosts go through Crawl4AI → Jina Reader fallback). Your agent learns one interface, not 15 APIs.
|
|
78
|
+
|
|
79
|
+
**3. Works even when WebSearch doesn't.**
|
|
80
|
+
On proxy / relay-station / Bedrock / Vertex-Claude-3.x setups where the built-in WebSearch server tool isn't available, omnireach gives search back — it runs entirely on the client side, bypassing both gate layers.
|
|
81
|
+
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
## How is this different from Agent-Reach?
|
|
85
|
+
|
|
86
|
+
[Agent-Reach](https://github.com/Panniantong/Agent-Reach) (51k★) created this category and is excellent at what it does. omnireach makes different trade-offs — these are date-stamped facts, not marketing:
|
|
87
|
+
|
|
88
|
+
| | omnireach | Agent-Reach (v1.5.0, as of 2026-07) |
|
|
89
|
+
|---|---|---|
|
|
90
|
+
| 微信公众号 WeChat | ✅ zero-config search (Sogou path) + full-text fetch via your logged-in Chrome | ❌ removed 2026-06 ([PR #347](https://github.com/Panniantong/Agent-Reach/pull/347)) after anti-bot breakage |
|
|
91
|
+
| 抖音 Douyin | ✅ search via your logged-in Chrome | ❌ removed 2026-06 (upstream tool archived) |
|
|
92
|
+
| TikTok | ✅ search | ❌ not supported |
|
|
93
|
+
| Output contract | one pydantic JSON schema across all 16 sources; auto-JSON when piped | no wrapper layer by design — each upstream tool's own format (YAML / plain text / subtitle files / raw JSON) |
|
|
94
|
+
| `search` / `fetch` commands | built-in: `omnireach search`, `omnireach fetch <url>` with host-aware routing | no search/read commands — routes your agent to call upstream tools directly |
|
|
95
|
+
| Facebook · Instagram · LinkedIn · 雪球 · podcast transcription | ❌ | ✅ |
|
|
96
|
+
| Community | early — you found us before the crowd | 51k★, 30 contributors |
|
|
97
|
+
|
|
98
|
+
Pick Agent-Reach if you need Facebook/LinkedIn/transcription. Pick omnireach if you need the Chinese internet — WeChat above all — a machine-stable JSON contract, or a single fetch entrypoint. Both MIT; they compose fine in one agent.
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## Example
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
# Search a login-walled vertical platform
|
|
106
|
+
omnireach search --on xiaohongshu --json "Claude Code 使用技巧"
|
|
107
|
+
|
|
108
|
+
# Fetch a WeChat article — login-walled, via your session
|
|
109
|
+
omnireach fetch --json "https://mp.weixin.qq.com/s/<token>"
|
|
110
|
+
|
|
111
|
+
# Full pipeline: search → fetch all results
|
|
112
|
+
omnireach search --on wechat --json "claude 4.7" \
|
|
113
|
+
| jq -r '.results[].url' \
|
|
114
|
+
| xargs -I{} omnireach fetch --json {}
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
---
|
|
118
|
+
|
|
119
|
+
## Commands
|
|
120
|
+
|
|
121
|
+
| Command | What it does |
|
|
122
|
+
|---|---|
|
|
123
|
+
| `omnireach search "<query>"` | Search (SERP: metadata + URL) |
|
|
124
|
+
| `omnireach search --on twitter,reddit "..."` | Target specific sources |
|
|
125
|
+
| `omnireach search --mode quick "..."` | Quick mode — HN only |
|
|
126
|
+
| `omnireach search --mode deep "..."` | Deep mode — all ready sources |
|
|
127
|
+
| `omnireach search --json "..."` | Explicit JSON output |
|
|
128
|
+
| **`omnireach fetch <url>`** (v0.10, v0.10.1) | **URL → full markdown** — `mp.weixin.qq.com` uses OpenCLI logged-in Chrome (v0.10.1+), other hosts use crwl → jina fallback |
|
|
129
|
+
| `omnireach fetch <url> --backend jina` | Force Jina Reader SaaS (zero local deps) |
|
|
130
|
+
| `omnireach fetch <url> --backend opencli` | Force OpenCLI wechat logged-in path (v0.10.1+) |
|
|
131
|
+
| `omnireach init` | Write default `~/.omnireach/preferences.toml` |
|
|
132
|
+
| `omnireach sources` | List all sources + tier status |
|
|
133
|
+
| `omnireach setup <source>` | Guided setup for a 🟡 / 🔴 source |
|
|
134
|
+
| `omnireach doctor` | Health check (sources / fetch backends / wechat backends) |
|
|
135
|
+
|
|
136
|
+
---
|
|
137
|
+
|
|
138
|
+
## Sources
|
|
139
|
+
|
|
140
|
+
| Source | Tier | Dependency | Notes |
|
|
141
|
+
|---|---|---|---|
|
|
142
|
+
| hackernews | ✅ ready | none | Direct Algolia, zero-config |
|
|
143
|
+
| youtube | ✅ ready | `yt-dlp` (pip install) | `omnireach setup youtube` |
|
|
144
|
+
| github | ✅ ready | `gh` CLI + `gh auth login` | `omnireach setup github` |
|
|
145
|
+
| rss | ✅ ready | built-in feedparser | query must be a URL |
|
|
146
|
+
| reddit | 🟡 one_step | `rdt-cli` + `rdt login` | `omnireach setup reddit` |
|
|
147
|
+
| twitter | 🔴 heavy | OpenCLI + Chrome extension | v0.3 path |
|
|
148
|
+
| xiaohongshu | 🔴 heavy | OpenCLI + Chrome extension | v0.3 path |
|
|
149
|
+
| tiktok | 🔴 heavy | OpenCLI + Chrome extension | TikTok international (v0.7) |
|
|
150
|
+
| douyin | 🔴 heavy | OpenCLI fork + Chrome extension | 抖音 (v0.7.2, Daily-AC/OpenCLI fork) |
|
|
151
|
+
| 💎 tavily | booster | env `TAVILY_API_KEY` | paid (v0.4) |
|
|
152
|
+
| 💎 brave | booster | env `BRAVE_API_KEY` | paid (v0.4) |
|
|
153
|
+
| 💎 perplexity | booster | env `PERPLEXITY_API_KEY` | paid (v0.4) |
|
|
154
|
+
| 💎 exa | booster | env `EXA_API_KEY` | paid web search (v0.5) |
|
|
155
|
+
| wechat | ✅ ready | none (optional `EXA_API_KEY` for enhancement) | WeChat Official Accounts — search via Sogou free path; `EXA_API_KEY` optionally enables semantic enhancement; v0.10.1+ `omnireach fetch <wechat-url>` auto-routes through OpenCLI logged-in Chrome |
|
|
156
|
+
| bilibili | ✅ ready | none (optional `EXA_API_KEY` for enhancement) | B站 — v0.9+ defaults to B站 official search API; `EXA_API_KEY` optionally enables semantic enhancement |
|
|
157
|
+
|
|
158
|
+
> **抖音 / douyin** (v0.7.2): Uses `omnireach setup douyin`, installs [Daily-AC/OpenCLI fork](https://github.com/Daily-AC/OpenCLI) (upstream PR [jackwener/OpenCLI#1759](https://github.com/jackwener/OpenCLI/pull/1759) pending review; will switch back once merged). Requires signing in to www.douyin.com in Chrome. `engagement.likes` has real data (DOM extraction); `plays/comments/shares` are not exposed in search cards, normalized to `null`.
|
|
159
|
+
|
|
160
|
+
> **WeChat fetch** (v0.10.1): `omnireach fetch <mp.weixin.qq.com URL>` uses the same Daily-AC/OpenCLI fork (`weixin download --stdout`, upstream PR [jackwener/OpenCLI#1770](https://github.com/jackwener/OpenCLI/pull/1770) pending). Requires having opened any mp.weixin.qq.com article in Chrome (no explicit login step; browser cookies suffice). See [How to get full text](#how-to-get-full-text-v08) below.
|
|
161
|
+
|
|
162
|
+
---
|
|
163
|
+
|
|
164
|
+
## Agent calling convention
|
|
165
|
+
|
|
166
|
+
When calling omnireach from an agent, **always request JSON explicitly** to prevent rich-table output from wrapping fields you need:
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
# Option 1: explicit flag per command
|
|
170
|
+
omnireach search --json "..."
|
|
171
|
+
omnireach fetch --json "<url>"
|
|
172
|
+
|
|
173
|
+
# Option 2: env var (recommended for agent harnesses)
|
|
174
|
+
export OMNIREACH_FORCE_JSON=1
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
The `not isatty()` auto-JSON added in v0.9.2 covers most cases, but some agent terminals (e.g. Antigravity) allocate a real PTY to subprocesses making `isatty()=True`. Explicit `--json` or the env var always works.
|
|
178
|
+
|
|
179
|
+
Full skill contract: [`.claude-plugin/skills/omnireach/SKILL.md`](./.claude-plugin/skills/omnireach/SKILL.md)
|
|
180
|
+
|
|
181
|
+
---
|
|
182
|
+
|
|
183
|
+
<details>
|
|
184
|
+
<summary><b>Who specifically needs this? (The WebSearch two-layer gate)</b></summary>
|
|
185
|
+
|
|
186
|
+
Claude Code's WebSearch is a **server-side server tool** (`web_search_20250305`). Whether it actually works depends on **two independent gates**:
|
|
187
|
+
|
|
188
|
+
**Gate 1 — client-side (`WebSearchTool.isEnabled()` checks `getAPIProvider()`):**
|
|
189
|
+
- Default `firstParty` (no `CLAUDE_CODE_USE_*` env var set, including the case where only `ANTHROPIC_BASE_URL` is changed) → tool **registered**
|
|
190
|
+
- Explicit `CLAUDE_CODE_USE_BEDROCK=1` → tool **off**
|
|
191
|
+
- Explicit `CLAUDE_CODE_USE_VERTEX=1` + Claude 4+ → registered; older Claude → off
|
|
192
|
+
- Explicit `CLAUDE_CODE_USE_FOUNDRY=1` → registered
|
|
193
|
+
|
|
194
|
+
**Gate 2 — upstream server tool implementation:** After the client sends the tool schema, the upstream API must have **specifically implemented** the `web_search_20250305` server tool (receive tool call → run search → return results to client):
|
|
195
|
+
- Real Anthropic API (api.anthropic.com): ✓ native
|
|
196
|
+
- Vertex / Foundry: ✓ (each backend implements it)
|
|
197
|
+
- **Third-party providers that specifically support Claude Code** (e.g. DeepSeek's Anthropic-compat endpoint): ✓ they implemented the server tool handling, routing to their own search backend
|
|
198
|
+
- **OpenAI-compatible relay stations** (cliproxy / anyrouter etc. that simply translate Claude API → OpenAI Chat Completions): ✗ don't recognize server tool semantics
|
|
199
|
+
- **Self-hosted gateways / most proxies**: ✗ generally not implemented
|
|
200
|
+
|
|
201
|
+
Gate 2 looks at **"did the upstream specifically implement Claude Code server tool support"** — not at "is this real Anthropic." Providers who specifically support Claude Code implement it; pure API translators don't. (Data point: DeepSeek is not real Anthropic but WebSearch works because they explicitly implemented it.)
|
|
202
|
+
|
|
203
|
+
**Root causes differ by scenario:**
|
|
204
|
+
- DeepSeek and other Claude-Code-specific third parties: both gates pass, WebSearch ✓ — omnireach value here is supplementing vertical sources (Twitter/小红书/微信), not patching search
|
|
205
|
+
- OpenAI-compatible relay station users: client registers the tool, **upstream doesn't handle the server tool call** → failure
|
|
206
|
+
- Explicit Bedrock users / Vertex Claude 3.x users: client-side `isEnabled` is off before the request even leaves
|
|
207
|
+
|
|
208
|
+
**Even with WebSearch working**, it can't reach Twitter threads, Reddit comment sections, 小红书 posts, WeChat articles, 抖音, B站 technical videos — these login-walled vertical platforms are blind spots for every hosted web search. omnireach's three value layers:
|
|
209
|
+
1. Patch for users whose client-side gate is off
|
|
210
|
+
2. Patch for users whose upstream doesn't implement the server tool
|
|
211
|
+
3. Supplement for users whose WebSearch works fine but can't reach vertical platforms
|
|
212
|
+
|
|
213
|
+
</details>
|
|
214
|
+
|
|
215
|
+
<details>
|
|
216
|
+
<summary><b>Naming & architecture (search / fetch / parse)</b></summary>
|
|
217
|
+
|
|
218
|
+
**omnireach** = `omni` (all) + `reach` (reach). Full "reach the whole web" semantics require three capability layers, all living as **sibling binaries in this repo** (analogous to `cargo` / `rustc` / `rustfmt` in the Rust monorepo — **no sister repo**):
|
|
219
|
+
|
|
220
|
+
| Layer | Implementation | Responsibility | Status |
|
|
221
|
+
|---|---|---|---|
|
|
222
|
+
| **search** | `omnireach search` subcommand | Locate across the web — returns metadata + URL, does not fetch content | ✅ v0.7+ in use |
|
|
223
|
+
| **fetch** | `omnireach fetch` subcommand | Given a URL, fetch full-text markdown — host-aware: `mp.weixin.qq.com` via OpenCLI logged-in Chrome, others via [Crawl4AI](https://github.com/unclecode/crawl4ai) → [Jina Reader](https://r.jina.ai/) | ✅ v0.10+ (wechat path v0.10.1+) |
|
|
224
|
+
| **parse** | (not yet implemented, future addition to this repo) | Video/audio content parsing (subtitles/STT/frame-by-frame) | 🔜 not started |
|
|
225
|
+
|
|
226
|
+
v0.10+ has omnireach covering search + fetch (subcommand form). Video parsing still uses external tools (yt-dlp / whisper); parse binary will be added to this repo when there are real user requests (YAGNI — no sister repo).
|
|
227
|
+
|
|
228
|
+
This split mirrors Anthropic's own [WebSearch](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/web-search) + [WebFetch](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/web-fetch) split: each layer does one thing well, search isn't slowed by parsing tasks, and agent callers can compose freely.
|
|
229
|
+
|
|
230
|
+
> Note: renaming `omnireach` to `omnisearch` was considered, but v0.10 landing `omnireach fetch` resolved the question — `omnireach search` (reach = find) + `omnireach fetch` (reach = retrieve) are both natural sub-actions of "reach," so the umbrella name fits. **No rename** (decided 2026-05-27).
|
|
231
|
+
|
|
232
|
+
</details>
|
|
233
|
+
|
|
234
|
+
<details>
|
|
235
|
+
<summary><b>Upgrade</b></summary>
|
|
236
|
+
|
|
237
|
+
omnireach is in alpha with frequent releases. Check and upgrade:
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
omnireach check-update # compare against GitHub Releases
|
|
241
|
+
uv tool install --force git+https://github.com/Daily-AC/omnireach.git # pull latest
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
> ⚠️ `uv tool upgrade omnireach` **will not** pull new commits (uv locks git-URL-installed tools to the commit at install time). `--force` reinstall fetches the latest.
|
|
245
|
+
|
|
246
|
+
</details>
|
|
247
|
+
|
|
248
|
+
<details>
|
|
249
|
+
<summary><b>Platform support</b></summary>
|
|
250
|
+
|
|
251
|
+
| Platform | Status | Notes |
|
|
252
|
+
|---|---|---|
|
|
253
|
+
| macOS | ✅ Primary dev platform | All sources tested (HN/RSS/youtube/github/reddit/twitter/xhs/tiktok/douyin + 4 boosters + wechat/bilibili); `omnireach fetch` all three backends (crwl/jina/opencli) verified |
|
|
254
|
+
| Linux | 🟡 best-effort | Should work; setup flow doesn't auto-detect `apt`/`pacman` |
|
|
255
|
+
| WSL2 | 🟡 best-effort | Same as Linux |
|
|
256
|
+
| Windows (native PowerShell) | 🟡 experimental (v0.6.3+) | macOS assumptions removed: secrets.env no longer calls POSIX chmod; preferences edit falls back to notepad; setup github prompts `winget install GitHub.cli`; OpenCLI sources (twitter/xhs) theoretically cross-platform but untested. **File an issue if you hit problems.** |
|
|
257
|
+
|
|
258
|
+
Run `omnireach doctor` to print a platform / Python version line at the top — useful to include when filing issues.
|
|
259
|
+
|
|
260
|
+
</details>
|
|
261
|
+
|
|
262
|
+
<details>
|
|
263
|
+
<summary><b>💎 Paid boosters (v0.4)</b></summary>
|
|
264
|
+
|
|
265
|
+
omnireach is free by default. Configuring a paid API key improves result quality:
|
|
266
|
+
|
|
267
|
+
```bash
|
|
268
|
+
omnireach setup tavily # guided key entry + writes to ~/.omnireach/secrets.env
|
|
269
|
+
omnireach setup brave
|
|
270
|
+
omnireach setup perplexity
|
|
271
|
+
omnireach setup exa # added v0.5 (replaces old `web` source)
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
Keys are detected automatically when set. Results carry `cost="paid"` metadata; TTY output shows a 💎 prefix for auditability.
|
|
275
|
+
|
|
276
|
+
To disable: edit `~/.omnireach/preferences.toml` and set `[boosters] auto_enable = false`.
|
|
277
|
+
|
|
278
|
+
</details>
|
|
279
|
+
|
|
280
|
+
<details>
|
|
281
|
+
<summary><b>⚙️ User preferences (v0.4)</b></summary>
|
|
282
|
+
|
|
283
|
+
`~/.omnireach/preferences.toml` can configure default sources, language, output format, and `source_trust` overrides.
|
|
284
|
+
|
|
285
|
+
```bash
|
|
286
|
+
omnireach preferences show # view current config
|
|
287
|
+
omnireach preferences edit # open in $EDITOR
|
|
288
|
+
omnireach preferences reset # reset (backs up original to .bak)
|
|
289
|
+
omnireach preferences path # print file location
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
</details>
|
|
293
|
+
|
|
294
|
+
<details>
|
|
295
|
+
<summary><b>How to get full text (v0.8)</b></summary>
|
|
296
|
+
|
|
297
|
+
omnireach is the search layer — the `content` field is uniformly capped at ≤ 500 characters + `…`. This validator runs in the contract layer (`SearchResult.content` pydantic `field_validator`) for **all sources**. This is intentional — full text belongs to the `omnireach fetch` layer (see [Naming & architecture](#naming--architecture-search--fetch--parse) above).
|
|
298
|
+
|
|
299
|
+
For sources whose upstream returns full text (wechat / exa / tavily) or long threads (twitter), the **complete raw payload is preserved in `result.raw`**, so agents that need full text can retrieve it:
|
|
300
|
+
|
|
301
|
+
```python
|
|
302
|
+
# Python (call CLI + parse JSON envelope)
|
|
303
|
+
import json
|
|
304
|
+
import subprocess
|
|
305
|
+
|
|
306
|
+
out = subprocess.run(
|
|
307
|
+
["omnireach", "search", "--json", "--on", "wechat", "claude 4.7"],
|
|
308
|
+
check=True, capture_output=True, text=True,
|
|
309
|
+
)
|
|
310
|
+
env = json.loads(out.stdout)
|
|
311
|
+
snippet = env["results"][0]["content"] # 500 chars + "…"
|
|
312
|
+
full = env["results"][0]["raw"]["text"] # Exa / wechat / twitter full text
|
|
313
|
+
# tavily uses raw["content"]
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
```bash
|
|
317
|
+
# CLI + jq
|
|
318
|
+
omnireach search --json --on tavily "claude 4.7" | \
|
|
319
|
+
jq '.results[] | {title, snippet: .content, full: .raw.content}'
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
Field mapping (verified by real E2E in v0.8.1 + v0.9):
|
|
323
|
+
|
|
324
|
+
| Source | `result.adapter` | `result.content` | `result.raw[...]` for full text / raw payload |
|
|
325
|
+
|---|---|---|---|
|
|
326
|
+
| wechat (default Sogou) | `sogou` | snippet (Sogou SERP summary) | `raw["item_html"]` (full Sogou card HTML) — for actual full text you need mp.weixin.qq.com |
|
|
327
|
+
| wechat (EXA_API_KEY enabled) | `exa-api` | snippet | `raw["text"]` (Exa full text) |
|
|
328
|
+
| bilibili (default B站 API) | `bilibili-api` | video description (≤500) | `raw` entire video item dict, includes desc/cover/aid/bvid |
|
|
329
|
+
| bilibili (EXA_API_KEY enabled) | `exa-api` | snippet | `raw["text"]` |
|
|
330
|
+
| exa | `exa-api` | snippet | `raw["text"]` |
|
|
331
|
+
| tavily | `tavily-api` | snippet | `raw["content"]` |
|
|
332
|
+
| twitter | `opencli` | snippet (long threads trigger truncation) | `raw["text"]` |
|
|
333
|
+
| xiaohongshu | `opencli` | empty — OpenCLI search results don't include body | n/a (no full text at search layer) |
|
|
334
|
+
|
|
335
|
+
The specific key names in `raw[...]` match upstream API schema directly — if upstream changes schema, this table needs updating. When uncertain, inspect `result.raw.keys()` first.
|
|
336
|
+
|
|
337
|
+
Other sources (HN / GitHub / RSS / YouTube / etc.) generally have content under 500 chars, so the validator is usually a no-op — but it's not guaranteed. For full-text fallback, check `result.raw` for a matching key.
|
|
338
|
+
|
|
339
|
+
### For actual full text → `omnireach fetch <url>` (v0.10+)
|
|
340
|
+
|
|
341
|
+
`omnireach fetch <url>` is the official search → full-text pipeline, with **host-aware** backend routing:
|
|
342
|
+
|
|
343
|
+
```bash
|
|
344
|
+
# Any webpage → crwl (local Crawl4AI) preferred, jina (r.jina.ai SaaS) fallback
|
|
345
|
+
omnireach fetch https://example.com/article --json
|
|
346
|
+
|
|
347
|
+
# WeChat Official Account article → automatically uses OpenCLI logged-in Chrome (v0.10.1+), bypasses CAPTCHA
|
|
348
|
+
omnireach fetch https://mp.weixin.qq.com/s/<token> --json
|
|
349
|
+
|
|
350
|
+
# search → fetch pipeline
|
|
351
|
+
omnireach search --on wechat "claude 4.7" --json \
|
|
352
|
+
| jq -r '.results[].url' \
|
|
353
|
+
| xargs -I{} omnireach fetch --json {}
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
Backend matrix:
|
|
357
|
+
|
|
358
|
+
| URL host | `--backend auto` routes to | Notes |
|
|
359
|
+
|---|---|---|
|
|
360
|
+
| `mp.weixin.qq.com` | **opencli** (logged-in cookie strategy) | Install [Daily-AC/OpenCLI fork](https://github.com/Daily-AC/OpenCLI) with `weixin download --stdout` flag (v0.10.1 commit fe28823+); direct `crwl` / `jina` are blocked by WeChat's "environment anomaly" CAPTCHA |
|
|
361
|
+
| Other hosts | **crwl → jina** | Crawl4AI preferred; falls back to Jina Reader SaaS if not installed or fails |
|
|
362
|
+
|
|
363
|
+
Explicit `--backend` overrides auto:
|
|
364
|
+
|
|
365
|
+
| Flag | Behavior |
|
|
366
|
+
|---|---|
|
|
367
|
+
| `--backend crwl` | Force Crawl4AI local (66K ⭐ Apache-2.0, built-in Cloudflare/Akamai/DataDome bypass) |
|
|
368
|
+
| `--backend jina` | Force [Jina Reader](https://r.jina.ai/) SaaS — zero local deps, large free tier |
|
|
369
|
+
| `--backend opencli` | Force OpenCLI wechat logged-in path (only meaningful for mp.weixin.qq.com) |
|
|
370
|
+
|
|
371
|
+
v0.10.1 adds **CAPTCHA heuristic fallback** for all backends: when keywords like `环境异常 / 完成验证后即可继续访问 / Cloudflare / Just a moment` are detected, the envelope `errors[]` gains a `captcha_suspected: ...` entry; the markdown field is preserved (graceful degrade — agent reads `errors` to decide whether to trust the content). `omnireach doctor`'s `wechat_backends` section surfaces whether OpenCLI + `--stdout` are ready.
|
|
372
|
+
|
|
373
|
+
</details>
|
|
374
|
+
|
|
375
|
+
<details>
|
|
376
|
+
<summary><b>Design</b></summary>
|
|
377
|
+
|
|
378
|
+
See `docs/superpowers/specs/2026-05-25-omnireach-design.md`.
|
|
379
|
+
|
|
380
|
+
</details>
|
|
381
|
+
|
|
382
|
+
<details>
|
|
383
|
+
<summary><b>License</b></summary>
|
|
384
|
+
|
|
385
|
+
MIT — see [LICENSE](LICENSE).
|
|
386
|
+
|
|
387
|
+
</details>
|