gitbook-downloader 11.0.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. gitbook_downloader-11.0.5/LICENSE +21 -0
  2. gitbook_downloader-11.0.5/PKG-INFO +555 -0
  3. gitbook_downloader-11.0.5/README.md +507 -0
  4. gitbook_downloader-11.0.5/pyproject.toml +86 -0
  5. gitbook_downloader-11.0.5/setup.cfg +4 -0
  6. gitbook_downloader-11.0.5/src/gitbook_downloader/__init__.py +27 -0
  7. gitbook_downloader-11.0.5/src/gitbook_downloader/__main__.py +5 -0
  8. gitbook_downloader-11.0.5/src/gitbook_downloader/api.py +636 -0
  9. gitbook_downloader-11.0.5/src/gitbook_downloader/cli.py +650 -0
  10. gitbook_downloader-11.0.5/src/gitbook_downloader/engine.py +691 -0
  11. gitbook_downloader-11.0.5/src/gitbook_downloader/gui/__init__.py +8 -0
  12. gitbook_downloader-11.0.5/src/gitbook_downloader/gui/app.py +65 -0
  13. gitbook_downloader-11.0.5/src/gitbook_downloader/gui/bridge.py +767 -0
  14. gitbook_downloader-11.0.5/src/gitbook_downloader/mcp/__init__.py +5 -0
  15. gitbook_downloader-11.0.5/src/gitbook_downloader/mcp/__main__.py +6 -0
  16. gitbook_downloader-11.0.5/src/gitbook_downloader/mcp/server.py +655 -0
  17. gitbook_downloader-11.0.5/src/gitbook_downloader/output_contract.py +370 -0
  18. gitbook_downloader-11.0.5/src/gitbook_downloader/providers/__init__.py +91 -0
  19. gitbook_downloader-11.0.5/src/gitbook_downloader/providers/base.py +274 -0
  20. gitbook_downloader-11.0.5/src/gitbook_downloader/providers/docusaurus.py +164 -0
  21. gitbook_downloader-11.0.5/src/gitbook_downloader/providers/generic.py +150 -0
  22. gitbook_downloader-11.0.5/src/gitbook_downloader/providers/gitbook.py +189 -0
  23. gitbook_downloader-11.0.5/src/gitbook_downloader/providers/mintlify.py +177 -0
  24. gitbook_downloader-11.0.5/src/gitbook_downloader/providers/mkdocs.py +215 -0
  25. gitbook_downloader-11.0.5/src/gitbook_downloader/providers/nextra.py +188 -0
  26. gitbook_downloader-11.0.5/src/gitbook_downloader/providers/readme.py +197 -0
  27. gitbook_downloader-11.0.5/src/gitbook_downloader/providers/readthedocs.py +173 -0
  28. gitbook_downloader-11.0.5/src/gitbook_downloader/providers/vitepress.py +190 -0
  29. gitbook_downloader-11.0.5/src/gitbook_downloader/search/__init__.py +6 -0
  30. gitbook_downloader-11.0.5/src/gitbook_downloader/search/graph.py +218 -0
  31. gitbook_downloader-11.0.5/src/gitbook_downloader/search/index.py +387 -0
  32. gitbook_downloader-11.0.5/src/gitbook_downloader/splitter.py +180 -0
  33. gitbook_downloader-11.0.5/src/gitbook_downloader/storage/__init__.py +6 -0
  34. gitbook_downloader-11.0.5/src/gitbook_downloader/storage/manager.py +752 -0
  35. gitbook_downloader-11.0.5/src/gitbook_downloader/storage/versioning.py +351 -0
  36. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/__init__.py +17 -0
  37. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/__main__.py +6 -0
  38. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/app.py +217 -0
  39. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/engine_protocol.py +209 -0
  40. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/real_engine.py +274 -0
  41. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/screens/__init__.py +15 -0
  42. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/screens/diagnostics.py +179 -0
  43. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/screens/diff.py +204 -0
  44. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/screens/library.py +202 -0
  45. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/screens/search.py +183 -0
  46. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/screens/wizard.py +553 -0
  47. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/testing.py +255 -0
  48. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/theme.py +169 -0
  49. gitbook_downloader-11.0.5/src/gitbook_downloader/tui/widgets.py +219 -0
  50. gitbook_downloader-11.0.5/src/gitbook_downloader/utils/__init__.py +13 -0
  51. gitbook_downloader-11.0.5/src/gitbook_downloader/utils/config.py +348 -0
  52. gitbook_downloader-11.0.5/src/gitbook_downloader/utils/discovery.py +298 -0
  53. gitbook_downloader-11.0.5/src/gitbook_downloader/utils/export.py +310 -0
  54. gitbook_downloader-11.0.5/src/gitbook_downloader/utils/renderer.py +95 -0
  55. gitbook_downloader-11.0.5/src/gitbook_downloader/utils/retry.py +100 -0
  56. gitbook_downloader-11.0.5/src/gitbook_downloader.egg-info/PKG-INFO +555 -0
  57. gitbook_downloader-11.0.5/src/gitbook_downloader.egg-info/SOURCES.txt +106 -0
  58. gitbook_downloader-11.0.5/src/gitbook_downloader.egg-info/dependency_links.txt +1 -0
  59. gitbook_downloader-11.0.5/src/gitbook_downloader.egg-info/entry_points.txt +4 -0
  60. gitbook_downloader-11.0.5/src/gitbook_downloader.egg-info/requires.txt +24 -0
  61. gitbook_downloader-11.0.5/src/gitbook_downloader.egg-info/top_level.txt +1 -0
  62. gitbook_downloader-11.0.5/tests/test_batch_run_button.py +51 -0
  63. gitbook_downloader-11.0.5/tests/test_bridge_contract.py +110 -0
  64. gitbook_downloader-11.0.5/tests/test_bridge_dead_alias.py +49 -0
  65. gitbook_downloader-11.0.5/tests/test_bridge_thread_safety.py +133 -0
  66. gitbook_downloader-11.0.5/tests/test_cli_banner.py +89 -0
  67. gitbook_downloader-11.0.5/tests/test_cli_rag.py +170 -0
  68. gitbook_downloader-11.0.5/tests/test_cli_rag_pdf_paths.py +61 -0
  69. gitbook_downloader-11.0.5/tests/test_command_menu_shortcuts.py +51 -0
  70. gitbook_downloader-11.0.5/tests/test_diff_view_snapshots.py +72 -0
  71. gitbook_downloader-11.0.5/tests/test_doc_reader_toast.py +63 -0
  72. gitbook_downloader-11.0.5/tests/test_engine_discovery.py +104 -0
  73. gitbook_downloader-11.0.5/tests/test_engine_flow.py +312 -0
  74. gitbook_downloader-11.0.5/tests/test_engine_providers.py +155 -0
  75. gitbook_downloader-11.0.5/tests/test_engine_url_identity.py +45 -0
  76. gitbook_downloader-11.0.5/tests/test_export_formats.py +84 -0
  77. gitbook_downloader-11.0.5/tests/test_fast_ast_removed.py +42 -0
  78. gitbook_downloader-11.0.5/tests/test_gui_bridge.py +103 -0
  79. gitbook_downloader-11.0.5/tests/test_gui_bridge_safety.py +149 -0
  80. gitbook_downloader-11.0.5/tests/test_imports.py +90 -0
  81. gitbook_downloader-11.0.5/tests/test_install_modal_opacity.py +44 -0
  82. gitbook_downloader-11.0.5/tests/test_mcp_advanced_tools.py +144 -0
  83. gitbook_downloader-11.0.5/tests/test_mcp_server.py +516 -0
  84. gitbook_downloader-11.0.5/tests/test_providers.py +481 -0
  85. gitbook_downloader-11.0.5/tests/test_providers_new.py +233 -0
  86. gitbook_downloader-11.0.5/tests/test_release_notes.py +414 -0
  87. gitbook_downloader-11.0.5/tests/test_search.py +474 -0
  88. gitbook_downloader-11.0.5/tests/test_shell_api.py +380 -0
  89. gitbook_downloader-11.0.5/tests/test_shell_cli.py +329 -0
  90. gitbook_downloader-11.0.5/tests/test_shell_config.py +200 -0
  91. gitbook_downloader-11.0.5/tests/test_shell_output_contract.py +281 -0
  92. gitbook_downloader-11.0.5/tests/test_shell_storage.py +342 -0
  93. gitbook_downloader-11.0.5/tests/test_splitter.py +318 -0
  94. gitbook_downloader-11.0.5/tests/test_stats_drift.py +123 -0
  95. gitbook_downloader-11.0.5/tests/test_storage.py +610 -0
  96. gitbook_downloader-11.0.5/tests/test_tui_app.py +209 -0
  97. gitbook_downloader-11.0.5/tests/test_tui_contract.py +221 -0
  98. gitbook_downloader-11.0.5/tests/test_tui_diagnostics.py +158 -0
  99. gitbook_downloader-11.0.5/tests/test_tui_diff.py +156 -0
  100. gitbook_downloader-11.0.5/tests/test_tui_fake_engine.py +90 -0
  101. gitbook_downloader-11.0.5/tests/test_tui_library.py +146 -0
  102. gitbook_downloader-11.0.5/tests/test_tui_paste.py +60 -0
  103. gitbook_downloader-11.0.5/tests/test_tui_search.py +166 -0
  104. gitbook_downloader-11.0.5/tests/test_tui_wizard.py +282 -0
  105. gitbook_downloader-11.0.5/tests/test_typo_classes.py +78 -0
  106. gitbook_downloader-11.0.5/tests/test_utils.py +615 -0
  107. gitbook_downloader-11.0.5/tests/test_version_drift.py +142 -0
  108. gitbook_downloader-11.0.5/tests/test_visual_anchors.py +130 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Rohan Shetty
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,555 @@
1
+ Metadata-Version: 2.4
2
+ Name: gitbook-downloader
3
+ Version: 11.0.5
4
+ Summary: DocHarvest โ€” Turn any documentation site into LLM-ready Markdown, vector RAG JSONL, llms.txt, and styled offline PDFs. Zero-config CLI, React desktop GUI & FastMCP server.
5
+ Author-email: Rohan Shetty <shettyrohan2@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/RohannShetty/gitbook-downloader
8
+ Project-URL: Repository, https://github.com/RohannShetty/gitbook-downloader
9
+ Project-URL: Issues, https://github.com/RohannShetty/gitbook-downloader/issues
10
+ Project-URL: Changelog, https://github.com/RohannShetty/gitbook-downloader/blob/main/CHANGELOG.md
11
+ Keywords: docharvest,gitbook,documentation,markdown,downloader,ai,rag,llms-txt,mcp-server,vector-db,docusaurus,readthedocs,mintlify,offline-docs,pdf-generator,nextra,vitepress,mkdocs,readme-io
12
+ Classifier: Development Status :: 5 - Production/Stable
13
+ Classifier: Environment :: Console
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Topic :: Internet :: WWW/HTTP
21
+ Classifier: Topic :: Software Development :: Documentation
22
+ Classifier: Topic :: Text Processing :: Markup
23
+ Classifier: Topic :: Utilities
24
+ Requires-Python: >=3.10
25
+ Description-Content-Type: text/markdown
26
+ License-File: LICENSE
27
+ Requires-Dist: requests>=2.28.0
28
+ Requires-Dist: beautifulsoup4>=4.11.0
29
+ Requires-Dist: markdownify>=0.11.0
30
+ Requires-Dist: lxml>=4.9.0
31
+ Requires-Dist: mcp>=1.2.0
32
+ Requires-Dist: textual>=0.60
33
+ Requires-Dist: pyperclip>=1.8.0
34
+ Requires-Dist: pywebview>=6.2.1
35
+ Requires-Dist: fpdf2>=2.8.8
36
+ Provides-Extra: mcp
37
+ Requires-Dist: mcp>=1.2.0; extra == "mcp"
38
+ Provides-Extra: render
39
+ Requires-Dist: playwright>=1.40.0; extra == "render"
40
+ Provides-Extra: js
41
+ Requires-Dist: playwright>=1.40.0; extra == "js"
42
+ Provides-Extra: dev
43
+ Requires-Dist: pytest>=8.0; extra == "dev"
44
+ Requires-Dist: pytest-timeout>=2.3; extra == "dev"
45
+ Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
46
+ Requires-Dist: pytest-cov>=4.0; extra == "dev"
47
+ Dynamic: license-file
48
+
49
+ <div align="center">
50
+
51
+ <img src="assets/logo-icon.svg" alt="DocHarvest Logo" width="96" />
52
+
53
+ # DocHarvest
54
+
55
+ ### Turn Any Documentation Site into LLM-Ready Markdown, Vector Context & Offline Books
56
+
57
+ **Zero-Config CLI ยท React Desktop GUI ยท Native FastMCP Server ยท Pure-Python PDF Studio**
58
+
59
+ [![Version: 11.0.5](https://img.shields.io/badge/version-11.0.5-06b6d4?style=flat-square&labelColor=090d16)](CHANGELOG.md)
60
+ [![License: MIT](https://img.shields.io/badge/license-MIT-10b981?style=flat-square&labelColor=090d16)](LICENSE)
61
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-3b82f6?style=flat-square&labelColor=090d16)](pyproject.toml)
62
+ [![UI: shadcn/ui](https://img.shields.io/badge/UI-shadcn%2Fui-27272a?style=flat-square&labelColor=090d16)](https://ui.shadcn.com)
63
+ [![Tests: 665 Passing](https://img.shields.io/badge/tests-665%20passing-10b981?style=flat-square&labelColor=090d16)](CHANGELOG.md)
64
+ [![PyPI](https://img.shields.io/pypi/v/gitbook-downloader?style=flat-square&labelColor=090d16&color=f59e0b)](https://pypi.org/project/gitbook-downloader/)
65
+ [![Showcase Website](https://img.shields.io/badge/website-Live%20Showcase-06b6d4?style=flat-square&labelColor=090d16)](https://rohannshetty.github.io/gitbook-downloader/)
66
+
67
+ <br />
68
+
69
+ <img src="assets/capture_studio.png" alt="DocHarvest Desktop GUI โ€” Capture Studio with shadcn/ui, 60fps motion progress, radial gauge, and live terminal logs" width="920" />
70
+
71
+ </div>
72
+
73
+ ---
74
+
75
+ ## โšก Overview
76
+
77
+ **Your coding agent doesn't read documentation โ€” it reads web pages.** Navbars, cookie banners, search modals, and footer scripts can make up 80โ€“85% of a raw page's bytes before a single API fact arrives. Chunks captured without source URLs make hallucinations unfalsifiable, and per-page cloud API bills spike the moment you index a real docs portal.
78
+
79
+ **DocHarvest** (package: `gitbook-downloader`) fixes that locally, in one command. It detects the documentation platform, bounds the crawl strictly to documentation subpaths, extracts clean markdown via direct `.md` endpoint probing and AST-based DOM cleaning, and compiles a deterministic, noise-free knowledge corpus โ€” measured at **~83% token reduction** on a real portal (full-suite reference capture: **673 pages in 18.2 seconds**).
80
+
81
+ Whether you are feeding 500-page API manuals to **Cursor / Claude Code**, building vector RAG pipelines with **LangChain & LlamaIndex**, reading offline on an airplane, or archiving technical libraries โ€” every capture ends in the same verifiable shape: clean markdown with SHA-256 provenance, ready for your agent or your bookshelf.
82
+
83
+ ---
84
+
85
+ ## โฑ๏ธ 30-Second Start
86
+
87
+ ```bash
88
+ pip install gitbook-downloader
89
+ docharvest capture https://docs.openalgo.in/ --rag --pdf
90
+ ```
91
+
92
+ No API key. No account. No telemetry. When the command finishes you own a `book.md`, an `llms.txt`, a RAG JSONL dataset, and a printable PDF โ€” all local, all MIT. Full install paths (standalone `.exe`, uvx, optional extras) are in [Quick Start](#-quick-start).
93
+
94
+ ---
95
+
96
+ ## ๐Ÿงฉ Supported Documentation Platforms (8 Real Providers)
97
+
98
+ DocHarvest features dedicated, priority-ordered parsers that extract clean article content and strip headers, footers, sidebars, anchor hashes, and cookie banners:
99
+
100
+ | Provider | Priority | Discovery Method | Clean Content Target |
101
+ | :--- | :---: | :--- | :--- |
102
+ | **GitBook** | `100` | `.md` endpoint probing, sitemap, space discovery | Native markdown or `.page-inner` / `article` |
103
+ | **Mintlify** | `90` | `mintlify.json`, OpenAPI specs, CDN asset anchors | `#content`, `#main-content`, `article` |
104
+ | **Docusaurus** | `80` | `sitemap.xml`, `docusaurus.config.js`, sidebars | `article`, `.markdown`, `main .theme-doc-markdown` |
105
+ | **Nextra** | `75` | `sitemap.xml`, Next.js app routes, nextra scripts | `main.nextra-content`, `article` |
106
+ | **VitePress** | `72` | Sitemap, VitePress theme anchors, route index | `div.vp-doc`, `div.VPContent`, `main.VPDoc` |
107
+ | **MkDocs** | `70` | `search/search_index.json`, sitemap | `article.md-content__inner`, `div.md-typeset` |
108
+ | **ReadMe.io** | `65` | `sitemap.xml`, `/llms.txt`, developer hub routes | `div.rm-Article`, `div.rm-Markdown`, `#content` |
109
+ | **ReadTheDocs** | `60` | Sphinx `sitemap.xml`, `div.sphinxsidebar` | `div.document[role="main"]`, `div.body` |
110
+ | **Generic HTML / SPA** | `0` | BFS link crawl, `llms.txt`, `sitemap.xml` | `main`, `article`, `[role="main"]`, `#content` |
111
+
112
+ > [!TIP]
113
+ > **Dynamic JavaScript SPAs**: If a site is rendered entirely client-side via JavaScript (such as `omp.sh/docs`), install the optional Playwright extra (`pip install "gitbook-downloader[render]" && playwright install chromium`) and run with `--render` to execute JavaScript before extracting markdown.
114
+
115
+ ---
116
+
117
+ ## ๐ŸŒŸ Key Capabilities
118
+
119
+ - ๐Ÿค– **Zero-Noise LLM Context**: Auto-detects 8 documentation frameworks, probes native `.md` endpoints, and cleans DOM trees โ€” measured at ~83% token reduction vs raw pages.
120
+ - ๐Ÿ“ฆ **Four-Part Output Contract**: Every capture yields a modular `pages/` tree with SHA-256 YAML frontmatter, a consolidated `book.md` with TOC, a standardized `llms.txt` manifest, and search index records.
121
+ - ๐Ÿš€ **Export Studio & Local Search**: RAG JSONL for vector databases, pure-Python PDF handbooks (`fpdf2`, zero C-dependencies), and AST markdown chunks โ€” all indexed into embedded SQLite FTS5 BM25 search.
122
+ - ๐Ÿ”Œ **Native FastMCP v2 Server**: 12 MCP tools plus resources and prompts over stdio, with ready-made configs for 14 AI clients (Cursor, Claude Code/Desktop, Windsurf, VS Code & more). Crash-safe atomic storage and semver snapshot diffing included.
123
+
124
+ ### What DocHarvest Is *Not* For
125
+
126
+ A tool that claims to do everything has earned none of your trust. Honest scope:
127
+
128
+ - **Not for login-walled, paywalled, or CAPTCHA-protected content.** DocHarvest is built for public technical documentation and will not bypass access controls.
129
+ - **Not for internet-scale crawling.** Millions of arbitrary URLs is Common Crawl / Scrapy territory; this is a documentation compiler, not a search-engine crawler.
130
+ - **Not for e-commerce or social feeds.** Product catalogs and social streams are out of scope by design.
131
+ - **Not a cloud service.** No dashboard, no subscription, no telemetry โ€” because nothing of yours ever leaves your machine.
132
+
133
+ ---
134
+
135
+ ## ๐Ÿ–ฅ๏ธ Desktop GUI: Document Library & Management
136
+
137
+ <div align="center">
138
+ <img src="assets/document_library.png" alt="DocHarvest Desktop GUI โ€” Document Library with search, rename, open folder, and multi-format exports" width="920" />
139
+ </div>
140
+
141
+ The desktop application includes a dedicated **Document Library** for managing all harvested technical docs:
142
+ - **Instant Search**: Real-time filtering across downloaded portals.
143
+ - **In-App Renaming**: Organize project labels without breaking file paths.
144
+ - **1-Click Export Studio**: Export selected docsets to Markdown, Vector JSONL, or PDF handbooks directly from the GUI.
145
+ - **Direct Finder/Explorer Integration**: Open raw markdown source folders with a single click.
146
+
147
+ ---
148
+
149
+ ## ๐Ÿ“‹ The Four-Part Output Contract
150
+
151
+ **What a generic crawler hands your LLM** (every page, every time):
152
+
153
+ ```html
154
+ <nav class="sidebar">โ€ฆ47 linksโ€ฆ</nav>
155
+ <div class="cookie-banner">We value your privacyโ€ฆ</div>
156
+ <main>
157
+ <h1>OAuth 2.0<a class="anchor" href="#oauth2">ยถ</a></h1>
158
+ <pre><code><span class="token-keyword">import</span> <span class="token-variable">requests</span>โ€ฆ</code></pre>
159
+ </main>
160
+ ```
161
+
162
+ **What DocHarvest delivers** โ€” the same page, with cryptographic provenance:
163
+
164
+ ````markdown
165
+ ---
166
+ source_url: https://docs.openalgo.in/v/v2.0/api-reference/oauth
167
+ title: "OAuth 2.0 Authentication"
168
+ content_hash: "sha256-2fa9ca2a57c4e974f1725657f88f757e25b90adee3e18ef809f65932d283746c"
169
+ ---
170
+
171
+ # OAuth 2.0 Authentication
172
+
173
+ ## Request Signature
174
+
175
+ ```python
176
+ import requests
177
+
178
+ response = requests.post(
179
+ "https://api.openalgo.in/oauth/token",
180
+ json={"client_id": "pk_live_..."},
181
+ )
182
+ ```
183
+ ````
184
+
185
+ The `content_hash` is the real SHA-256 of the markdown body shown above โ€” paste it into any SHA-256 tool and it verifies.
186
+
187
+ Every crawl produces the same standardized, deterministic directory structure:
188
+
189
+ ```
190
+ ~/.gitbook-downloader/docs/
191
+ โ””โ”€โ”€ docs.openalgo.in/
192
+ โ”œโ”€โ”€ pages/ # Modular individual markdown files
193
+ โ”‚ โ”œโ”€โ”€ 001_quickstart.md
194
+ โ”‚ โ””โ”€โ”€ 002_api_reference.md
195
+ โ”œโ”€โ”€ book.md # Consolidated single handbook with hierarchical TOC
196
+ โ”œโ”€โ”€ llms.txt # Standardized AI discovery manifest
197
+ โ”œโ”€โ”€ exports/
198
+ โ”‚ โ”œโ”€โ”€ openalgo_rag.jsonl # Tokenized vector chunks + metadata
199
+ โ”‚ โ””โ”€โ”€ openalgo_handbook.pdf # Publication-grade printable PDF (pure Python)
200
+ โ””โ”€โ”€ .manifest.json # Crawl metadata, engine version & cryptographic hashes
201
+ ```
202
+
203
+ ---
204
+
205
+ ## ๐Ÿš€ Quick Start
206
+
207
+ ### Option 1: Standalone Executable (Zero Setup)
208
+ Download **[`docharvest-windows-latest.exe`](https://github.com/RohannShetty/gitbook-downloader/releases/latest)** from the latest release:
209
+ - **Double-click** to launch the **Desktop GUI Application**.
210
+ - Or execute directly in your terminal:
211
+ ```powershell
212
+ .\docharvest-windows-latest.exe crawl https://docs.openalgo.in/ --rag --pdf
213
+ ```
214
+
215
+ ### Option 2: Install via pip / PyPI
216
+ ```bash
217
+ # Standard installation (100% local, zero C-dependencies)
218
+ pip install gitbook-downloader
219
+
220
+ # Optional headless browser rendering for dynamic JavaScript SPAs
221
+ pip install "gitbook-downloader[render]"
222
+ playwright install chromium
223
+
224
+ # MCP server: the FastMCP SDK ships in the base install โ€” nothing extra needed.
225
+ # (The [mcp] extra is a backward-compatibility no-op; this line still resolves.)
226
+ pip install "gitbook-downloader[mcp]"
227
+
228
+ # Launch desktop GUI:
229
+ docharvest --gui
230
+
231
+ # Or run a CLI crawl:
232
+ docharvest capture https://docs.openalgo.in/ --rag --pdf
233
+ ```
234
+
235
+ ### Option 3: Ultra-Fast One-Liner via uv / uvx
236
+ ```bash
237
+ # Launch GUI instantly without permanent installation:
238
+ uvx gitbook-downloader --gui
239
+
240
+ # Or install as a global CLI tool:
241
+ uv tool install gitbook-downloader
242
+ ```
243
+
244
+ ---
245
+
246
+ ## ๐Ÿ’ป CLI Command Reference
247
+
248
+ ```bash
249
+ # Basic Documentation Crawl (aliases: capture, dl, crawl)
250
+ # `docharvest capture` is the canonical verb; `crawl` remains as a documented alias.
251
+ docharvest capture https://docs.openalgo.in/
252
+
253
+ # Full Compilation (Markdown + RAG JSONL + llms.txt + PDF Handbook)
254
+ docharvest capture https://docs.openalgo.in/ --rag --pdf
255
+
256
+ # Crawl Dynamic Client-Rendered SPAs (Playwright Headless Browser)
257
+ docharvest capture https://omp.sh/docs --render
258
+
259
+ # Restrict Crawl to Specific Path Prefix & Limit Depth
260
+ docharvest capture https://docs.example.com/ --scope /api/ --max-pages 50
261
+
262
+ # Full-Text BM25 Search across Harvested Docs
263
+ docharvest search "OAuth 2.0 authentication token"
264
+
265
+ # List Harvested Document Domains in Local Library
266
+ docharvest ls
267
+
268
+ # Show Snapshot History & Diff Versions
269
+ docharvest history docs.example.com
270
+ docharvest diff docs.example.com v1.0.0 v1.0.1
271
+
272
+ # Start FastMCP Server over Stdio for AI IDEs
273
+ docharvest --mcp
274
+
275
+ # Launch Desktop GUI Application
276
+ docharvest --gui
277
+ ```
278
+
279
+ ---
280
+
281
+ ## ๐Ÿ”Œ AI Agent Integration: Native FastMCP v2 Server
282
+
283
+ DocHarvest includes a native **FastMCP (Model Context Protocol v2)** server that exposes 12 high-level tools, **MCP Resources**, and **MCP Prompts** over standard input/output (`stdio`). It is compatible with both `mcp<2` and `mcp>=2.1`.
284
+
285
+ > **The `mcp` SDK ships in the base install.** `pip install gitbook-downloader` (or `uvx gitbook-downloader mcp`) is enough โ€” no extras required. The `gitbook-downloader[mcp]` extra is still accepted for backward compatibility but is now a no-op.
286
+
287
+ ### All 12 Native MCP Tools
288
+
289
+ 1. `download_docs(url, max_pages=None, workers=8, path_scope=[], exclude_paths=[], site_versions=None, output_mode="both")`
290
+ Captures any documentation URL into Markdown, `book.md`, and `llms.txt`.
291
+ 2. `search_docs(query, domain=None, limit=10)`
292
+ Full-text search across downloaded documentation via SQLite FTS5 BM25.
293
+ 3. `find_docs(query, limit=10)`
294
+ Resolves library/framework names ("react", "nextjs") to indexed domains in the local library.
295
+ 4. `read_doc(domain, path=None, topic=None, max_tokens=4000, version=None)`
296
+ Reads a specific page or topic section with AST-safe token bounding โ€” code blocks and tables are never split.
297
+ 5. `get_doc(domain, version=None)`
298
+ Retrieves the compiled documentation content or preview for a domain.
299
+ 6. `list_domains()`
300
+ Returns metadata for all harvested documentation portals in local storage.
301
+ 7. `query_doc_graph(domain, query, limit=10)`
302
+ Queries the semantic entity & concept graph to discover connected API endpoints and sections without reading full files.
303
+ 8. `get_related_concepts(domain, concept)`
304
+ Returns 1-hop and 2-hop connected concepts and prerequisite sections.
305
+ 9. `diff_versions(domain, v1, v2)`
306
+ Computes unified diffs and line change statistics between two snapshots.
307
+ 10. `list_versions(domain)`
308
+ Lists available captured snapshots and timestamps for a domain.
309
+ 11. `export_docs(domain, format="markdown")`
310
+ Exports documentation into `"markdown"`, `"jsonl"`, or `"rag"` metadata formats.
311
+ 12. `get_changelog(domain)`
312
+ Auto-generates version changelogs across captured snapshot iterations.
313
+
314
+ ### MCP v2 Resources & Prompts
315
+ - **Resources**: `docs://{domain}/book` (full handbook), `docs://{domain}/manifest` (`llms.txt` index).
316
+ - **Prompts**: `prompt://search-docset` (guided docset synthesis), `prompt://summarize-library` (library overview).
317
+
318
+ ---
319
+
320
+ ### IDE & Agent Configuration Matrix (14 Clients)
321
+
322
+ *The three most common clients are shown inline โ€” expand the list for all 14.*
323
+
324
+ #### 1. Claude Code
325
+ ```bash
326
+ claude mcp add docharvest docharvest mcp
327
+ ```
328
+ Or in `~/.claude.json`:
329
+ ```json
330
+ {
331
+ "mcpServers": {
332
+ "docharvest": {
333
+ "command": "docharvest",
334
+ "args": ["mcp"]
335
+ }
336
+ }
337
+ }
338
+ ```
339
+
340
+ #### 2. Claude Desktop (`claude_desktop_config.json`)
341
+ ```json
342
+ {
343
+ "mcpServers": {
344
+ "docharvest": {
345
+ "command": "uvx",
346
+ "args": ["gitbook-downloader", "mcp"]
347
+ }
348
+ }
349
+ }
350
+ ```
351
+
352
+ #### 3. Cursor (`.cursor/mcp.json`)
353
+ ```json
354
+ {
355
+ "mcpServers": {
356
+ "docharvest": {
357
+ "command": "python",
358
+ "args": ["-m", "gitbook_downloader.mcp"]
359
+ }
360
+ }
361
+ }
362
+ ```
363
+
364
+ <details>
365
+ <summary><strong>11 more client configs โ€” Windsurf ยท VS Code ยท JetBrains ยท Zed ยท Cline ยท Continue ยท Kiro ยท OpenCode ยท Pi/Oh My Pi ยท Gemini CLI ยท Codex CLI</strong></summary>
366
+
367
+ #### 4. Windsurf (`~/.codeium/windsurf/mcp_config.json`)
368
+ ```json
369
+ {
370
+ "mcpServers": {
371
+ "docharvest": {
372
+ "command": "docharvest",
373
+ "args": ["mcp"]
374
+ }
375
+ }
376
+ }
377
+ ```
378
+
379
+ #### 5. VS Code (`.vscode/mcp.json`)
380
+ ```json
381
+ {
382
+ "servers": {
383
+ "docharvest": {
384
+ "type": "stdio",
385
+ "command": "docharvest",
386
+ "args": ["mcp"]
387
+ }
388
+ }
389
+ }
390
+ ```
391
+
392
+ #### 6. JetBrains AI Assistant / PyCharm / IntelliJ
393
+ Configure via **Settings โ†’ Tools โ†’ Model Context Protocol (MCP)**:
394
+ - **Server Name**: `docharvest`
395
+ - **Command**: `docharvest`
396
+ - **Arguments**: `mcp`
397
+
398
+ #### 7. Zed (`settings.json`)
399
+ ```json
400
+ {
401
+ "context_servers": {
402
+ "docharvest": {
403
+ "command": "docharvest",
404
+ "args": ["mcp"]
405
+ }
406
+ }
407
+ }
408
+ ```
409
+
410
+ #### 8. Cline (`cline_mcp_settings.json`)
411
+ ```json
412
+ {
413
+ "mcpServers": {
414
+ "docharvest": {
415
+ "command": "docharvest",
416
+ "args": ["mcp"],
417
+ "disabled": false,
418
+ "autoApprove": ["search_docs", "get_doc", "list_domains", "query_doc_graph"]
419
+ }
420
+ }
421
+ }
422
+ ```
423
+
424
+ #### 9. Continue.dev (`config.json`)
425
+ ```json
426
+ {
427
+ "experimental": {
428
+ "modelContextProtocolServers": [
429
+ {
430
+ "transport": {
431
+ "type": "stdio",
432
+ "command": "docharvest",
433
+ "args": ["mcp"]
434
+ }
435
+ }
436
+ ]
437
+ }
438
+ }
439
+ ```
440
+
441
+ #### 10. Kiro (`.kiro/settings/mcp.json`)
442
+ ```json
443
+ {
444
+ "mcp": {
445
+ "servers": {
446
+ "docharvest": {
447
+ "command": "docharvest",
448
+ "args": ["mcp"]
449
+ }
450
+ }
451
+ }
452
+ }
453
+ ```
454
+
455
+ #### 11. OpenCode (`opencode.json`)
456
+ ```json
457
+ {
458
+ "mcp": {
459
+ "docharvest": {
460
+ "command": "docharvest",
461
+ "args": ["mcp"]
462
+ }
463
+ }
464
+ }
465
+ ```
466
+
467
+ #### 12. Pi (`pi.dev`) / Oh My Pi (`omp.sh`) (`~/.omp/config.json`)
468
+ ```json
469
+ {
470
+ "mcp_servers": {
471
+ "docharvest": {
472
+ "command": "docharvest",
473
+ "args": ["mcp"]
474
+ }
475
+ }
476
+ }
477
+ ```
478
+
479
+ #### 13. Antigravity / Gemini CLI (`mcp/docharvest.json`)
480
+ ```json
481
+ {
482
+ "name": "docharvest",
483
+ "command": "docharvest",
484
+ "args": ["mcp"]
485
+ }
486
+ ```
487
+
488
+ #### 14. OpenAI Codex CLI (`codex_config.json`)
489
+ ```json
490
+ {
491
+ "mcp_servers": {
492
+ "docharvest": {
493
+ "command": "docharvest",
494
+ "args": ["mcp"]
495
+ }
496
+ }
497
+ }
498
+ ```
499
+
500
+ </details>
501
+
502
+ ---
503
+
504
+ ## ๐Ÿ Python SDK Example
505
+
506
+ ```python
507
+ from gitbook_downloader.api import capture, CaptureOptions
508
+
509
+ # Configure capture options
510
+ options = CaptureOptions(
511
+ workers=8,
512
+ output_mode="both", # "both", "library", or "local"
513
+ render=False, # set True for dynamic JavaScript SPAs
514
+ )
515
+
516
+ # Execute deterministic capture
517
+ result = capture("https://docs.openalgo.in/", options=options)
518
+
519
+ print(f"Captured {result.pages_captured} pages using provider: {result.provider}")
520
+ print(f"Book file: {result.book_file}")
521
+ print(f"Manifest: {result.manifest_file}")
522
+ ```
523
+
524
+ ---
525
+
526
+ ## ๐Ÿงช Test Suite & Quality Benchmarks
527
+
528
+ DocHarvest is continuously tested across Windows, Linux, and macOS:
529
+
530
+ - **686 Automated Tests**: 100% pass rate across engine discovery, BFS crawling, provider extraction, storage safety, DocGraph semantic search, and MCP v2 tools (verified on this release).
531
+ - **73%+ Statement Coverage**: Rigorous test suites covering error recovery, invalid signatures, domain locks, and AST link normalization.
532
+ - **Windows CRLF Safe**: All link and boilerplate stripping routines are cross-platform normalized against Windows CRLF and Unix LF linebreaks.
533
+
534
+ To run the test suite locally:
535
+ ```bash
536
+ uv run pytest --cov=gitbook_downloader
537
+ ```
538
+
539
+ ---
540
+
541
+ ## ๐Ÿ‘จโ€๐Ÿ’ป Author & Connect
542
+
543
+ Created with โค๏ธ by **Rohan Shetty**.
544
+
545
+ - ๐ŸŒ **Website**: [rohannshetty.github.io/gitbook-downloader](https://rohannshetty.github.io/gitbook-downloader/)
546
+ - ๐Ÿ™ **GitHub**: [@RohannShetty](https://github.com/RohannShetty)
547
+ - ๐Ÿ’ผ **LinkedIn**: [linkedin.com/in/rohan-shettyy](https://www.linkedin.com/in/rohan-shettyy/)
548
+ - ๐Ÿฆ **X (Twitter)**: [@rohan__shetty](https://x.com/rohan__shetty)
549
+ - ๐Ÿ“ง **Email**: [shettyrohan2@gmail.com](mailto:shettyrohan2@gmail.com)
550
+
551
+ ---
552
+
553
+ ## ๐Ÿ“„ License
554
+
555
+ This project is licensed under the [MIT License](LICENSE).