graphgrail 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. graphgrail-0.1.0/.gitignore +62 -0
  2. graphgrail-0.1.0/LICENSE +21 -0
  3. graphgrail-0.1.0/PKG-INFO +68 -0
  4. graphgrail-0.1.0/configs/community.yaml +33 -0
  5. graphgrail-0.1.0/configs/embeddings.yaml +19 -0
  6. graphgrail-0.1.0/configs/endpoints.yaml +76 -0
  7. graphgrail-0.1.0/configs/grail.yaml +24 -0
  8. graphgrail-0.1.0/configs/indexing.yaml +62 -0
  9. graphgrail-0.1.0/configs/llm.yaml +24 -0
  10. graphgrail-0.1.0/configs/prompts.yaml +10 -0
  11. graphgrail-0.1.0/configs/reranker.yaml +33 -0
  12. graphgrail-0.1.0/configs/search.yaml +35 -0
  13. graphgrail-0.1.0/configs/storage.yaml +14 -0
  14. graphgrail-0.1.0/configs/templates/README.md +101 -0
  15. graphgrail-0.1.0/configs/templates/low_cost_setup/community.yaml +45 -0
  16. graphgrail-0.1.0/configs/templates/low_cost_setup/embeddings.yaml +29 -0
  17. graphgrail-0.1.0/configs/templates/low_cost_setup/endpoints.yaml +38 -0
  18. graphgrail-0.1.0/configs/templates/low_cost_setup/grail.yaml +18 -0
  19. graphgrail-0.1.0/configs/templates/low_cost_setup/indexing.yaml +85 -0
  20. graphgrail-0.1.0/configs/templates/low_cost_setup/llm.yaml +29 -0
  21. graphgrail-0.1.0/configs/templates/low_cost_setup/prompts.yaml +14 -0
  22. graphgrail-0.1.0/configs/templates/low_cost_setup/reranker.yaml +10 -0
  23. graphgrail-0.1.0/configs/templates/low_cost_setup/search.yaml +63 -0
  24. graphgrail-0.1.0/configs/templates/low_cost_setup/storage.yaml +14 -0
  25. graphgrail-0.1.0/configs/templates/low_cost_setup/vectorstore.yaml +19 -0
  26. graphgrail-0.1.0/configs/vectorstore.yaml +15 -0
  27. graphgrail-0.1.0/docs/benchmarks.md +156 -0
  28. graphgrail-0.1.0/docs/cli_chat.md +338 -0
  29. graphgrail-0.1.0/docs/comparison.md +323 -0
  30. graphgrail-0.1.0/docs/config.md +100 -0
  31. graphgrail-0.1.0/docs/getting-started.md +144 -0
  32. graphgrail-0.1.0/docs/glossary.md +232 -0
  33. graphgrail-0.1.0/docs/incremental_pipeline.md +544 -0
  34. graphgrail-0.1.0/docs/indexing.md +155 -0
  35. graphgrail-0.1.0/docs/llm.md +140 -0
  36. graphgrail-0.1.0/docs/model_selection/README.md +135 -0
  37. graphgrail-0.1.0/docs/model_selection/report.md +475 -0
  38. graphgrail-0.1.0/docs/preprocessing.md +145 -0
  39. graphgrail-0.1.0/docs/prompt_customization.md +865 -0
  40. graphgrail-0.1.0/docs/prompts.md +116 -0
  41. graphgrail-0.1.0/docs/python_api.md +370 -0
  42. graphgrail-0.1.0/docs/query.md +548 -0
  43. graphgrail-0.1.0/docs/reranker.md +121 -0
  44. graphgrail-0.1.0/docs/search_modes.md +698 -0
  45. graphgrail-0.1.0/docs/storage.md +123 -0
  46. graphgrail-0.1.0/docs/ui_frontend_analysis.md +176 -0
  47. graphgrail-0.1.0/docs/vectorstores.md +70 -0
  48. graphgrail-0.1.0/docs/viz.md +423 -0
  49. graphgrail-0.1.0/grail/__init__.py +59 -0
  50. graphgrail-0.1.0/grail/_version.py +1 -0
  51. graphgrail-0.1.0/grail/apps/__init__.py +0 -0
  52. graphgrail-0.1.0/grail/apps/chat/__init__.py +0 -0
  53. graphgrail-0.1.0/grail/apps/chat/auth.py +69 -0
  54. graphgrail-0.1.0/grail/apps/chat/database.py +349 -0
  55. graphgrail-0.1.0/grail/apps/chat/frontend/dist/assets/index-2a40wZqj.css +1 -0
  56. graphgrail-0.1.0/grail/apps/chat/frontend/dist/assets/index-DfRgtpKs.js +239 -0
  57. graphgrail-0.1.0/grail/apps/chat/frontend/dist/index.html +17 -0
  58. graphgrail-0.1.0/grail/apps/chat/frontend/index.html +16 -0
  59. graphgrail-0.1.0/grail/apps/chat/frontend/package-lock.json +3990 -0
  60. graphgrail-0.1.0/grail/apps/chat/frontend/package.json +29 -0
  61. graphgrail-0.1.0/grail/apps/chat/frontend/src/App.tsx +76 -0
  62. graphgrail-0.1.0/grail/apps/chat/frontend/src/components/ChatInput.tsx +131 -0
  63. graphgrail-0.1.0/grail/apps/chat/frontend/src/components/ChatView.tsx +96 -0
  64. graphgrail-0.1.0/grail/apps/chat/frontend/src/components/InfoPanel.tsx +119 -0
  65. graphgrail-0.1.0/grail/apps/chat/frontend/src/components/Layout.tsx +118 -0
  66. graphgrail-0.1.0/grail/apps/chat/frontend/src/components/LoginView.tsx +199 -0
  67. graphgrail-0.1.0/grail/apps/chat/frontend/src/components/MarkdownRenderer.tsx +227 -0
  68. graphgrail-0.1.0/grail/apps/chat/frontend/src/components/MessageBubble.tsx +237 -0
  69. graphgrail-0.1.0/grail/apps/chat/frontend/src/components/ModeSelector.tsx +189 -0
  70. graphgrail-0.1.0/grail/apps/chat/frontend/src/components/Sidebar.tsx +211 -0
  71. graphgrail-0.1.0/grail/apps/chat/frontend/src/components/WelcomeView.tsx +182 -0
  72. graphgrail-0.1.0/grail/apps/chat/frontend/src/index.css +166 -0
  73. graphgrail-0.1.0/grail/apps/chat/frontend/src/lib/api.ts +162 -0
  74. graphgrail-0.1.0/grail/apps/chat/frontend/src/lib/store.ts +395 -0
  75. graphgrail-0.1.0/grail/apps/chat/frontend/src/main.tsx +10 -0
  76. graphgrail-0.1.0/grail/apps/chat/frontend/src/vite-env.d.ts +1 -0
  77. graphgrail-0.1.0/grail/apps/chat/frontend/tsconfig.app.json +21 -0
  78. graphgrail-0.1.0/grail/apps/chat/frontend/tsconfig.app.tsbuildinfo +1 -0
  79. graphgrail-0.1.0/grail/apps/chat/frontend/tsconfig.json +7 -0
  80. graphgrail-0.1.0/grail/apps/chat/frontend/tsconfig.node.json +19 -0
  81. graphgrail-0.1.0/grail/apps/chat/frontend/tsconfig.node.tsbuildinfo +1 -0
  82. graphgrail-0.1.0/grail/apps/chat/frontend/vite.config.ts +13 -0
  83. graphgrail-0.1.0/grail/apps/chat/schemas.py +90 -0
  84. graphgrail-0.1.0/grail/apps/chat/server.py +414 -0
  85. graphgrail-0.1.0/grail/apps/cli_chat/__init__.py +8 -0
  86. graphgrail-0.1.0/grail/apps/cli_chat/app.py +990 -0
  87. graphgrail-0.1.0/grail/apps/cli_chat/commands.py +181 -0
  88. graphgrail-0.1.0/grail/apps/cli_chat/reporter.py +73 -0
  89. graphgrail-0.1.0/grail/apps/cli_chat/widgets.py +496 -0
  90. graphgrail-0.1.0/grail/cli/__init__.py +7 -0
  91. graphgrail-0.1.0/grail/cli/banner.py +819 -0
  92. graphgrail-0.1.0/grail/cli/main.py +1663 -0
  93. graphgrail-0.1.0/grail/config.py +548 -0
  94. graphgrail-0.1.0/grail/core.py +1064 -0
  95. graphgrail-0.1.0/grail/export/__init__.py +5 -0
  96. graphgrail-0.1.0/grail/export/neo4j.py +298 -0
  97. graphgrail-0.1.0/grail/indexing/__init__.py +36 -0
  98. graphgrail-0.1.0/grail/indexing/communities.py +214 -0
  99. graphgrail-0.1.0/grail/indexing/community_reports.py +486 -0
  100. graphgrail-0.1.0/grail/indexing/entities_relationships.py +978 -0
  101. graphgrail-0.1.0/grail/indexing/entity_dedup.py +396 -0
  102. graphgrail-0.1.0/grail/indexing/incremental_community.py +382 -0
  103. graphgrail-0.1.0/grail/indexing/leiden.py +177 -0
  104. graphgrail-0.1.0/grail/indexing/loader.py +612 -0
  105. graphgrail-0.1.0/grail/indexing/preprocess.py +312 -0
  106. graphgrail-0.1.0/grail/indexing/run_manifest.py +274 -0
  107. graphgrail-0.1.0/grail/indexing/schema_migration.py +114 -0
  108. graphgrail-0.1.0/grail/indexing/stable_lcc.py +42 -0
  109. graphgrail-0.1.0/grail/indexing/summarize_descriptions.py +249 -0
  110. graphgrail-0.1.0/grail/llm/__init__.py +39 -0
  111. graphgrail-0.1.0/grail/llm/cache.py +122 -0
  112. graphgrail-0.1.0/grail/llm/cost.py +253 -0
  113. graphgrail-0.1.0/grail/llm/embeddings.py +228 -0
  114. graphgrail-0.1.0/grail/llm/providers.py +198 -0
  115. graphgrail-0.1.0/grail/llm/reranker.py +220 -0
  116. graphgrail-0.1.0/grail/llm/wrapper.py +654 -0
  117. graphgrail-0.1.0/grail/memory/__init__.py +55 -0
  118. graphgrail-0.1.0/grail/memory/_merge.py +288 -0
  119. graphgrail-0.1.0/grail/memory/analyses/__init__.py +70 -0
  120. graphgrail-0.1.0/grail/memory/analyses/alias_detect.py +190 -0
  121. graphgrail-0.1.0/grail/memory/analyses/edge_density.py +164 -0
  122. graphgrail-0.1.0/grail/memory/analyses/folder_split.py +135 -0
  123. graphgrail-0.1.0/grail/memory/analyses/membership.py +118 -0
  124. graphgrail-0.1.0/grail/memory/consolidate.py +164 -0
  125. graphgrail-0.1.0/grail/memory/identity.py +231 -0
  126. graphgrail-0.1.0/grail/memory/observation.py +165 -0
  127. graphgrail-0.1.0/grail/memory/project.py +1805 -0
  128. graphgrail-0.1.0/grail/memory/proposals.py +240 -0
  129. graphgrail-0.1.0/grail/memory/types.py +127 -0
  130. graphgrail-0.1.0/grail/prompts/__init__.py +25 -0
  131. graphgrail-0.1.0/grail/prompts/builtin/__init__.py +8 -0
  132. graphgrail-0.1.0/grail/prompts/builtin/claim_extraction.py +107 -0
  133. graphgrail-0.1.0/grail/prompts/builtin/community_report.py +170 -0
  134. graphgrail-0.1.0/grail/prompts/builtin/create_custom_entities.py +138 -0
  135. graphgrail-0.1.0/grail/prompts/builtin/entity_dedup.py +117 -0
  136. graphgrail-0.1.0/grail/prompts/builtin/entity_relation.py +267 -0
  137. graphgrail-0.1.0/grail/prompts/builtin/global_map.py +114 -0
  138. graphgrail-0.1.0/grail/prompts/builtin/global_reduce.py +77 -0
  139. graphgrail-0.1.0/grail/prompts/builtin/json_correction.py +83 -0
  140. graphgrail-0.1.0/grail/prompts/builtin/local_search.py +73 -0
  141. graphgrail-0.1.0/grail/prompts/builtin/summarize_description.py +105 -0
  142. graphgrail-0.1.0/grail/prompts/loader.py +172 -0
  143. graphgrail-0.1.0/grail/query/__init__.py +29 -0
  144. graphgrail-0.1.0/grail/query/agent.py +560 -0
  145. graphgrail-0.1.0/grail/query/cascade_search.py +376 -0
  146. graphgrail-0.1.0/grail/query/document_search.py +294 -0
  147. graphgrail-0.1.0/grail/query/global_search.py +287 -0
  148. graphgrail-0.1.0/grail/query/local_search.py +331 -0
  149. graphgrail-0.1.0/grail/query/recall_filter.py +312 -0
  150. graphgrail-0.1.0/grail/query/recall_search.py +264 -0
  151. graphgrail-0.1.0/grail/query/retrieval.py +425 -0
  152. graphgrail-0.1.0/grail/query/trace.py +119 -0
  153. graphgrail-0.1.0/grail/reporting/__init__.py +7 -0
  154. graphgrail-0.1.0/grail/reporting/rich_reporter.py +183 -0
  155. graphgrail-0.1.0/grail/schemas.py +382 -0
  156. graphgrail-0.1.0/grail/storage/__init__.py +29 -0
  157. graphgrail-0.1.0/grail/storage/base.py +127 -0
  158. graphgrail-0.1.0/grail/storage/local.py +116 -0
  159. graphgrail-0.1.0/grail/storage/s3.py +118 -0
  160. graphgrail-0.1.0/grail/utils/__init__.py +19 -0
  161. graphgrail-0.1.0/grail/utils/chunker.py +47 -0
  162. graphgrail-0.1.0/grail/utils/ids.py +19 -0
  163. graphgrail-0.1.0/grail/utils/text.py +44 -0
  164. graphgrail-0.1.0/grail/utils/tokens.py +29 -0
  165. graphgrail-0.1.0/grail/utils/zip.py +50 -0
  166. graphgrail-0.1.0/grail/vectorstores/__init__.py +29 -0
  167. graphgrail-0.1.0/grail/vectorstores/base.py +75 -0
  168. graphgrail-0.1.0/grail/vectorstores/chroma.py +145 -0
  169. graphgrail-0.1.0/grail/vectorstores/faiss.py +158 -0
  170. graphgrail-0.1.0/grail/vectorstores/lancedb.py +111 -0
  171. graphgrail-0.1.0/grail/viz/__init__.py +24 -0
  172. graphgrail-0.1.0/grail/viz/builder.py +129 -0
  173. graphgrail-0.1.0/grail/viz/colors.py +127 -0
  174. graphgrail-0.1.0/grail/viz/exporter.py +552 -0
  175. graphgrail-0.1.0/grail/viz/layout.py +315 -0
  176. graphgrail-0.1.0/grail/viz/template.py +1015 -0
  177. graphgrail-0.1.0/pyproject.toml +109 -0
  178. graphgrail-0.1.0/skills/grail/INSTALL.md +101 -0
  179. graphgrail-0.1.0/skills/grail/SKILL.md +149 -0
  180. graphgrail-0.1.0/skills/grail/agents/openai.yaml +20 -0
  181. graphgrail-0.1.0/skills/grail/assets/grail.kb.yaml.tpl +26 -0
  182. graphgrail-0.1.0/skills/grail/assets/grail.memory.yaml.tpl +44 -0
  183. graphgrail-0.1.0/skills/grail/assets/observation.md.tpl +14 -0
  184. graphgrail-0.1.0/skills/grail/references/config_reference.md +107 -0
  185. graphgrail-0.1.0/skills/grail/references/kb_mode.md +81 -0
  186. graphgrail-0.1.0/skills/grail/references/memory_mode.md +130 -0
  187. graphgrail-0.1.0/skills/grail/references/memory_tools.md +126 -0
  188. graphgrail-0.1.0/skills/grail/references/proposals.md +102 -0
  189. graphgrail-0.1.0/skills/grail/references/query_optimization.md +79 -0
  190. graphgrail-0.1.0/skills/grail/references/search_modes.md +94 -0
  191. graphgrail-0.1.0/skills/grail/references/troubleshooting.md +103 -0
  192. graphgrail-0.1.0/skills/grail/requirements.txt +9 -0
  193. graphgrail-0.1.0/skills/grail/scripts/_common.py +227 -0
  194. graphgrail-0.1.0/skills/grail/scripts/append.py +87 -0
  195. graphgrail-0.1.0/skills/grail/scripts/delete.py +52 -0
  196. graphgrail-0.1.0/skills/grail/scripts/edit.py +75 -0
  197. graphgrail-0.1.0/skills/grail/scripts/env_check.py +52 -0
  198. graphgrail-0.1.0/skills/grail/scripts/explore.py +94 -0
  199. graphgrail-0.1.0/skills/grail/scripts/index.py +83 -0
  200. graphgrail-0.1.0/skills/grail/scripts/init_project.py +108 -0
  201. graphgrail-0.1.0/skills/grail/scripts/list_grail_projects.py +53 -0
  202. graphgrail-0.1.0/skills/grail/scripts/memory/add_community.py +70 -0
  203. graphgrail-0.1.0/skills/grail/scripts/memory/add_entity.py +67 -0
  204. graphgrail-0.1.0/skills/grail/scripts/memory/add_observation.py +109 -0
  205. graphgrail-0.1.0/skills/grail/scripts/memory/add_relationship.py +62 -0
  206. graphgrail-0.1.0/skills/grail/scripts/memory/apply_proposal.py +56 -0
  207. graphgrail-0.1.0/skills/grail/scripts/memory/consolidate.py +54 -0
  208. graphgrail-0.1.0/skills/grail/scripts/memory/find_similar_entity.py +51 -0
  209. graphgrail-0.1.0/skills/grail/scripts/memory/list_proposals.py +49 -0
  210. graphgrail-0.1.0/skills/grail/scripts/memory/recall.py +76 -0
  211. graphgrail-0.1.0/skills/grail/scripts/query.py +151 -0
  212. graphgrail-0.1.0/skills/grail/scripts/setup.sh +24 -0
  213. graphgrail-0.1.0/skills/grail/scripts/status.py +97 -0
@@ -0,0 +1,62 @@
1
+ # --- Legacy/proprietary import baseline ---
2
+ # The _legacy_source/ tree is a verbatim copy of the proprietary `nirvana_agents/utilities/graphrag`
3
+ # snapshot used as the starting point for the GRAIL migration. It is intentionally kept in-tree
4
+ # so Claude Code sessions are self-contained (no path dependency on the proprietary monorepo).
5
+ # It must NEVER be committed to the public GRAIL repo. Delete the directory once migration is complete.
6
+ _legacy_source/
7
+
8
+ # --- Python ---
9
+ __pycache__/
10
+ *.py[cod]
11
+ *$py.class
12
+ *.so
13
+ .Python
14
+ *.egg
15
+ *.egg-info/
16
+ build/
17
+ /dist/
18
+ .eggs/
19
+ pip-wheel-metadata/
20
+ .pytest_cache/
21
+ .mypy_cache/
22
+ .ruff_cache/
23
+ .tox/
24
+ .coverage
25
+ .coverage.*
26
+ htmlcov/
27
+ .hypothesis/
28
+
29
+ # --- Virtual envs ---
30
+ .venv/
31
+ venv/
32
+ env/
33
+ ENV/
34
+
35
+ # --- Node (frontend builds) ---
36
+ node_modules/
37
+
38
+ # --- IDE / OS ---
39
+ .idea/
40
+ .vscode/
41
+ .DS_Store
42
+ Thumbs.db
43
+
44
+ # --- GRAIL runtime artifacts (default working dir) ---
45
+ # Outputs of indexing pipelines (parquet, lancedb, graphml). User overrides via config.
46
+ output/
47
+ cache/
48
+ lancedb/
49
+ *.parquet
50
+ *.graphml
51
+ mapping.json
52
+
53
+ # --- Notebook generated output ---
54
+ notebooks/_pyvis_graph.html
55
+ notebooks/.ipynb_checkpoints/
56
+
57
+ # --- Secrets / env ---
58
+ .env
59
+ .env.*
60
+ !.env.example
61
+ *.pem
62
+ *.key
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Cámara Chilena de Inteligencia Artificial
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,68 @@
1
+ Metadata-Version: 2.4
2
+ Name: graphgrail
3
+ Version: 0.1.0
4
+ Summary: Graph RAG with Advanced Integration and Learning — an open-source GraphRAG library with incremental updates, file-level provenance, and pluggable LLM/storage backends.
5
+ Project-URL: Repository, https://github.com/CAMARA-CHILENA-INTELIGENCIA-ARTIFICIAL/GRAIL
6
+ Author-email: Benjamin González Guerrero <ben@nirvana-ai.com>
7
+ License: Apache-2.0
8
+ License-File: LICENSE
9
+ Keywords: graphrag,knowledge-graph,leiden,llm,rag,retrieval
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: OSI Approved :: Apache Software License
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
18
+ Requires-Python: >=3.10
19
+ Requires-Dist: aiohttp>=3.9
20
+ Requires-Dist: aiosqlite>=0.20
21
+ Requires-Dist: graspologic>=3.4
22
+ Requires-Dist: httpx>=0.27
23
+ Requires-Dist: lancedb>=0.10
24
+ Requires-Dist: networkx>=3.2
25
+ Requires-Dist: numpy>=1.24
26
+ Requires-Dist: openai<2,>=1.40
27
+ Requires-Dist: pandas>=2.0
28
+ Requires-Dist: pyarrow>=14
29
+ Requires-Dist: pydantic<3,>=2.5
30
+ Requires-Dist: pypdf>=5.0
31
+ Requires-Dist: python-docx>=1.1
32
+ Requires-Dist: python-dotenv>=1.0
33
+ Requires-Dist: pyyaml>=6
34
+ Requires-Dist: rich>=13
35
+ Requires-Dist: scikit-learn>=1.3
36
+ Requires-Dist: scipy>=1.10
37
+ Requires-Dist: tenacity>=8.2
38
+ Requires-Dist: textual>=0.50
39
+ Requires-Dist: tiktoken>=0.7
40
+ Requires-Dist: typer>=0.12
41
+ Provides-Extra: chroma
42
+ Requires-Dist: chromadb>=0.5; extra == 'chroma'
43
+ Provides-Extra: dev
44
+ Requires-Dist: mypy>=1.10; extra == 'dev'
45
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
46
+ Requires-Dist: pytest-mock>=3.12; extra == 'dev'
47
+ Requires-Dist: pytest>=8; extra == 'dev'
48
+ Requires-Dist: ruff>=0.5; extra == 'dev'
49
+ Provides-Extra: faiss
50
+ Requires-Dist: faiss-cpu>=1.7; extra == 'faiss'
51
+ Provides-Extra: notebook
52
+ Requires-Dist: ipykernel>=6.29; extra == 'notebook'
53
+ Requires-Dist: ipywidgets>=8.1; extra == 'notebook'
54
+ Requires-Dist: jupyter>=1.0; extra == 'notebook'
55
+ Requires-Dist: matplotlib>=3.8; extra == 'notebook'
56
+ Requires-Dist: pyvis>=0.3.2; extra == 'notebook'
57
+ Provides-Extra: rerank
58
+ Requires-Dist: sentence-transformers>=2.7; extra == 'rerank'
59
+ Provides-Extra: s3
60
+ Requires-Dist: aioboto3>=12; extra == 's3'
61
+ Requires-Dist: boto3>=1.34; extra == 's3'
62
+ Provides-Extra: ui
63
+ Requires-Dist: bcrypt==4.0.1; extra == 'ui'
64
+ Requires-Dist: fastapi>=0.115; extra == 'ui'
65
+ Requires-Dist: passlib[bcrypt]>=1.7; extra == 'ui'
66
+ Requires-Dist: python-jose[cryptography]>=3.3; extra == 'ui'
67
+ Requires-Dist: sse-starlette>=2.0; extra == 'ui'
68
+ Requires-Dist: uvicorn[standard]>=0.30; extra == 'ui'
@@ -0,0 +1,33 @@
1
+ # Community detection + reporting config. See docs/community.md.
2
+
3
+ # Hierarchical Leiden (graspologic).
4
+ max_cluster_size: 50
5
+ use_lcc: false
6
+ strategy: leiden
7
+ seed: null # set an int for reproducible clustering
8
+
9
+ # Per-stage endpoint+model overrides. When null, inherit from configs/llm.yaml.
10
+ community_report_endpoint: null
11
+ community_report_model: null
12
+ json_corrector_endpoint: null
13
+ json_corrector_model: null
14
+
15
+ # JSON community report ceiling.
16
+ max_report_length: 8192
17
+
18
+ # Concurrency for community report LLM calls. null = use the global
19
+ # llm.concurrent_requests limit. Set lower to avoid overwhelming the
20
+ # endpoint with heavy report-generation calls.
21
+ # report_concurrency: null
22
+
23
+ # If true, include claims (covariates) in the report context.
24
+ include_covariates: false
25
+
26
+ # Incremental updates — change ratio threshold above which we re-cluster
27
+ # the affected subgraph instead of label-propagating new nodes.
28
+ incremental_change_threshold: 0.3
29
+
30
+ # After Leiden, small communities below this size get merged with their nearest
31
+ # centroid via DBSCAN over the entity embeddings.
32
+ min_community_size: 10
33
+ embedding_merge_eps: 0.5
@@ -0,0 +1,19 @@
1
+ # Embedding client config. See docs/llm.md.
2
+ #
3
+ # Same shape as configs/llm.yaml. endpoint references entries in
4
+ # configs/endpoints.yaml. The defaults below use DeepInfra's hosting of
5
+ # intfloat/multilingual-e5-large — a strong open-weights multilingual embedder.
6
+
7
+ endpoint: deepinfra
8
+ model: intfloat/multilingual-e5-large
9
+ encoding_format: float
10
+
11
+ # Batching / concurrency
12
+ max_batch_size: 1024
13
+ concurrent_requests: 30
14
+
15
+ # Timeouts / retries
16
+ request_timeout: 180.0
17
+ max_retries: 10
18
+ max_retry_wait: 10.0
19
+ sleep_on_rate_limit: 30.0
@@ -0,0 +1,76 @@
1
+ # Endpoint definitions. See docs/llm.md.
2
+ #
3
+ # Every entry here is a deployment of the OpenAI Chat Completions / Embeddings
4
+ # protocol. GRAIL is endpoint-agnostic by design — there are no vendor-specific
5
+ # code paths. To use a different vLLM cluster, sglang server, regional OpenAI
6
+ # proxy, or anything else, add an entry below and reference it by name in
7
+ # llm.yaml / embeddings.yaml / per-stage overrides.
8
+ #
9
+ # These defaults match the built-in registry in grail/llm/providers.py. User
10
+ # edits MERGE with the defaults (per-endpoint deep merge), so you only need to
11
+ # include entries you actually want to override.
12
+
13
+ openai:
14
+ base_url: https://api.openai.com/v1
15
+ api_key_env: OPENAI_API_KEY
16
+
17
+ anthropic:
18
+ base_url: https://api.anthropic.com/v1
19
+ api_key_env: ANTHROPIC_API_KEY
20
+ notes: Anthropic exposes /v1/messages over the OpenAI-compatible protocol.
21
+
22
+ deepinfra:
23
+ base_url: https://api.deepinfra.com/v1/openai
24
+ api_key_env: DEEPINFRA_API_KEY
25
+
26
+ together:
27
+ base_url: https://api.together.xyz/v1
28
+ api_key_env: TOGETHER_API_KEY
29
+
30
+ groq:
31
+ base_url: https://api.groq.com/openai/v1
32
+ api_key_env: GROQ_API_KEY
33
+
34
+ openrouter:
35
+ base_url: https://openrouter.ai/api/v1
36
+ api_key_env: OPENROUTER_API_KEY
37
+
38
+ # --- Self-hosted endpoints (no key by default) ---
39
+
40
+ ollama:
41
+ base_url: http://localhost:11434/v1
42
+ api_key_env: OLLAMA_API_KEY
43
+ requires_key: false
44
+ notes: Local Ollama server.
45
+
46
+ vllm:
47
+ base_url: http://localhost:8000/v1
48
+ api_key_env: VLLM_API_KEY
49
+ requires_key: false
50
+ notes: Self-hosted vLLM server speaking OpenAI's protocol.
51
+
52
+ sglang:
53
+ base_url: http://localhost:30000/v1
54
+ api_key_env: SGLANG_API_KEY
55
+ requires_key: false
56
+ notes: Self-hosted SGLang server.
57
+
58
+ lmstudio:
59
+ base_url: http://localhost:1234/v1
60
+ api_key_env: LMSTUDIO_API_KEY
61
+ requires_key: false
62
+ notes: LM Studio local server.
63
+
64
+ local:
65
+ base_url: http://localhost:8000/v1
66
+ api_key_env: LOCAL_API_KEY
67
+ requires_key: false
68
+ notes: Catch-all entry for arbitrary local servers.
69
+
70
+ # --- Add your own deployments below ---
71
+ #
72
+ # my-internal-vllm:
73
+ # base_url: https://vllm.internal.corp/v1
74
+ # api_key_env: INTERNAL_VLLM_KEY
75
+ # requires_key: true
76
+ # notes: Behind the corporate VPN.
@@ -0,0 +1,24 @@
1
+ # GRAIL master config.
2
+ # Loaded by `grail.config.load_config(path)`. Sibling per-module YAMLs in this
3
+ # directory are auto-merged into the matching section, so you can split overrides
4
+ # by concern. Documentation for every key lives in docs/<module>.md.
5
+
6
+ project_name: example
7
+ # Resolves with ~ expansion. All on-disk artefacts (parquets, lancedb, cache) live here.
8
+ root_dir: ~/.grail/projects/example
9
+
10
+ # Each block below mirrors a Pydantic submodel in grail/config.py. Anything you omit
11
+ # falls back to the default in code. To split this file by module, leave the section
12
+ # empty here and drop e.g. configs/llm.yaml next to this one — both load and merge.
13
+
14
+ llm: {}
15
+ embeddings: {}
16
+ indexing: {}
17
+ community: {}
18
+ search: {}
19
+ storage:
20
+ backend: local
21
+ # If unset, defaults to {root_dir}.
22
+ root: ~/.grail/projects/example
23
+ prompts: {}
24
+ vectorstore: {}
@@ -0,0 +1,62 @@
1
+ # Indexing pipeline config. See docs/indexing.md.
2
+
3
+ # Chunking
4
+ chunk_size: 2000
5
+ chunk_overlap: 50
6
+ encoding_name: cl100k_base
7
+ document_boundary: "\n\n---DOCUMENT_BOUNDARY---\n\n"
8
+
9
+ # Layout under storage root.
10
+ input_folder: input
11
+ output_folder: output
12
+ cache_folder: cache
13
+
14
+ # Per-stage endpoint+model overrides. When null, inherit from configs/llm.yaml.
15
+ # Set one or both per stage — e.g. cheap extraction on Groq, headline summarization on OpenAI.
16
+ entity_relation_endpoint: null
17
+ entity_relation_model: null
18
+ summarization_endpoint: null
19
+ summarization_model: null
20
+
21
+ # Entity types normalized to UPPER_SNAKE_CASE. Minimum 5 required.
22
+ # Always include PERSON and ORGANIZATION as baseline types.
23
+ # Add domain-specific types, or use discover_entity_types to let the LLM propose them.
24
+ entity_types:
25
+ - person
26
+ - organization
27
+ - location
28
+ - event
29
+ - concept
30
+
31
+ # When true, the LLM reads a corpus sample before extraction and proposes
32
+ # additional entity types (merged with the list above, capped at max_entity_types).
33
+ # Also available as CLI flag: grail index --discover-entities
34
+ # discover_entity_types: false
35
+ # max_entity_types: 15
36
+
37
+ # Extraction delimiters — the entity/relationship parser is contract-bound to
38
+ # these tokens. Change them only if you have a custom entity_relation prompt
39
+ # that uses different delimiters. When absent, the built-in defaults apply.
40
+ # tuple_delimiter: "<|>"
41
+ # record_delimiter: "##"
42
+ # completion_delimiter: "<|COMPLETE|>"
43
+ # start_delimiter: "<|START_OUTPUT|>"
44
+
45
+ # --- Token budgets per LLM call ---
46
+ # extraction_max_tokens: max output tokens for each entity/relationship extraction call.
47
+ # entity_discovery_max_tokens: max output tokens for the create-entities discovery call.
48
+ # max_summarization_tokens: max tokens for entity description summarization.
49
+ extraction_max_tokens: 8192
50
+ entity_discovery_max_tokens: 2048
51
+ max_summarization_tokens: 756
52
+
53
+ # --- Per-stage concurrency ---
54
+ # Override the global llm.concurrent_requests for specific stages.
55
+ # null = use the global limit. Useful to run more extraction calls in parallel
56
+ # (cheap, fast) while throttling heavier community-report generation.
57
+ # extraction_concurrency: null
58
+ # summarization_concurrency: null
59
+
60
+ # Gleanings: the upstream graphrag setting that re-asks the LLM to find missed
61
+ # entities. 0 = disabled. Costs roughly N× more tokens per chunk.
62
+ max_gleanings: 0
@@ -0,0 +1,24 @@
1
+ # LLM client config. See docs/llm.md.
2
+ #
3
+ # Endpoint and model are separate. `endpoint` is a name referenced in
4
+ # configs/endpoints.yaml (defaults cover openai, anthropic, deepinfra, together,
5
+ # groq, openrouter, ollama, vllm, sglang, lmstudio, local).
6
+
7
+ # Which deployment of the OpenAI protocol to use by default.
8
+ endpoint: openai
9
+ # Model name within that endpoint. Whatever string the endpoint accepts.
10
+ model: gpt-4o-mini
11
+
12
+ # Concurrency / timeouts
13
+ concurrent_requests: 15
14
+ request_timeout: 180.0
15
+ max_retries: 10
16
+ max_retry_wait: 10.0
17
+ sleep_on_rate_limit: 30.0
18
+
19
+ debug: false
20
+
21
+ # Disk cache. When enabled, identical (endpoint, model, messages, params) tuples
22
+ # are reused. Useful during iteration.
23
+ cache_enabled: false
24
+ cache_dir: null # null → {root_dir}/cache/llm
@@ -0,0 +1,10 @@
1
+ # Prompts config. See docs/prompts.md.
2
+
3
+ # Extra directories searched (in order) before grail/prompts/builtin/. Each file
4
+ # in these directories must expose NAME, REQUIRED_PARAMS, build_messages(**params).
5
+ custom_paths: []
6
+
7
+ # strict=true requires that the custom directory provide every built-in prompt
8
+ # (entity_relation, summarize_description, community_report, json_correction,
9
+ # create_custom_entities, local_search, global_map, global_reduce, claim_extraction).
10
+ strict: false
@@ -0,0 +1,33 @@
1
+ # Reranker config (optional). See docs/reranker.md.
2
+ #
3
+ # When enabled, a cross-encoder re-ranker scores entity and text-unit candidates
4
+ # after the initial vector similarity retrieval. This improves which items fill
5
+ # the limited context window sent to the LLM, at the cost of one extra API call.
6
+ #
7
+ # Toggle per-query with --rerank / --no-rerank on the CLI, or use_reranker= in
8
+ # the Python API, without changing this config.
9
+
10
+ enabled: false
11
+
12
+ # Endpoint name (from endpoints.yaml) that hosts the reranker model.
13
+ # The reranker uses a dedicated HTTP API (not OpenAI Chat Completions).
14
+ endpoint: deepinfra
15
+
16
+ # Cross-encoder model name. Passed to the inference API.
17
+ model: Qwen/Qwen3-Reranker-0.6B
18
+
19
+ # Full URL override. When null, auto-derived from endpoint name.
20
+ # DeepInfra: https://api.deepinfra.com/v1/inference/{model}
21
+ base_url: null
22
+
23
+ # Over-fetch factor: retrieve top_k * overfetch_factor entities by vector
24
+ # similarity, then rerank and trim to top_k. Higher = more candidates for the
25
+ # reranker to choose from, but more tokens sent to the reranker API.
26
+ overfetch_factor: 3
27
+
28
+ # Which retrieval stages to rerank.
29
+ rerank_entities: true
30
+ rerank_text_units: true
31
+
32
+ # HTTP timeout for the reranker API call (seconds).
33
+ request_timeout: 30.0
@@ -0,0 +1,35 @@
1
+ # Search config. See docs/query.md.
2
+
3
+ # Per-stage endpoint+model overrides. When null, inherit from configs/llm.yaml.
4
+ local_search_endpoint: null
5
+ local_search_model: null
6
+ global_search_endpoint: null
7
+ global_search_model: null
8
+
9
+ # --- Local search budgeting ---
10
+ local_max_tokens: 12000
11
+ local_text_unit_prop: 0.5
12
+ local_community_prop: 0.1
13
+ local_conversation_history_max_turns: 5
14
+ local_top_k_entities: 10
15
+ local_top_k_relationships: 10
16
+
17
+ # When false (default), community reports include full LLM-generated content
18
+ # with detailed findings. When true, only the one-line summary is used (legacy
19
+ # behaviour for tight token budgets or small-context models).
20
+ use_community_summary: false
21
+
22
+ # --- Global search budgeting ---
23
+ global_map_max_tokens: 2000
24
+ global_reduce_max_tokens: 8192
25
+ # When the assembled community-report context exceeds this token budget, GRAIL
26
+ # falls back to a map-reduce path. Otherwise it does a single reduce pass.
27
+ global_chunk_size: 100000
28
+ global_concurrency: 5
29
+
30
+ # Maximum tokens the LLM can generate in its response.
31
+ response_max_tokens: 16384
32
+
33
+ # Description of the desired response style — injected into the global reduce
34
+ # prompt's artifact_instructions block.
35
+ response_type: "Multiple Paragraphs"
@@ -0,0 +1,14 @@
1
+ # Storage backend config. See docs/storage.md.
2
+
3
+ # "local" (default) writes everything under `root` on the filesystem.
4
+ # "s3" writes objects under `s3_bucket/s3_prefix` (requires the [s3] extra).
5
+ backend: local
6
+
7
+ # Local backend
8
+ root: ~/.grail/projects/default
9
+
10
+ # S3 backend (only used when backend=s3)
11
+ s3_bucket: null
12
+ s3_prefix: null
13
+ s3_region: null
14
+ s3_endpoint_url: null # set for MinIO / S3-compatible stores
@@ -0,0 +1,101 @@
1
+ # GRAIL configuration templates
2
+
3
+ A **template** is a directory of YAML files matching the GRAIL config layout
4
+ (``grail.yaml`` + optional ``endpoints.yaml``, ``llm.yaml``, ``embeddings.yaml``,
5
+ ``indexing.yaml``, ``community.yaml``, ``search.yaml``, ``storage.yaml``,
6
+ ``prompts.yaml``, ``vectorstore.yaml``).
7
+
8
+ When you scaffold a project with ``grail init <project_dir> --template NAME``,
9
+ every recognised YAML in that template directory is copied into the new project
10
+ with ``{name}``, ``{root}``, and ``{date}`` placeholders substituted.
11
+
12
+ ## The "everything is OpenAI API" position
13
+
14
+ GRAIL only speaks the OpenAI Chat Completions + Embeddings protocol. An
15
+ **endpoint** is just one deployment of that protocol — a base URL the SDK posts
16
+ to. Any compatible host plugs in: OpenAI itself, DeepInfra, Together, Groq,
17
+ your own vLLM / SGLang / LM Studio server.
18
+
19
+ The base URL is the ``/v1`` prefix. The SDK appends ``/chat/completions`` or
20
+ ``/embeddings`` automatically. **Do not include ``/chat/completions`` or
21
+ ``/embeddings`` in ``base_url``.**
22
+
23
+ Examples of valid ``base_url`` values:
24
+
25
+ | Host | base_url |
26
+ |-----------------------|------------------------------------------------|
27
+ | OpenAI | ``https://api.openai.com/v1`` |
28
+ | DeepInfra | ``https://api.deepinfra.com/v1/openai`` |
29
+ | Together | ``https://api.together.xyz/v1`` |
30
+ | Groq | ``https://api.groq.com/openai/v1`` |
31
+ | Local vLLM | ``http://localhost:8000/v1`` |
32
+ | Local SGLang | ``http://localhost:30000/v1`` |
33
+
34
+ ## Built-in templates
35
+
36
+ ### `low_cost_setup` — the one we ship
37
+
38
+ A fully-filled GRAIL project biased for cost:
39
+
40
+ - One inference endpoint pointed at DeepInfra (swap the ``base_url`` to retarget).
41
+ - Conservative concurrency (8 in-flight calls).
42
+ - LLM disk cache **on** so re-running indexing is free.
43
+ - Single-pass extraction (``max_gleanings: 0``).
44
+ - Tighter search token budgets than the GRAIL defaults.
45
+
46
+ Files included:
47
+
48
+ ```
49
+ configs/templates/low_cost_setup/
50
+ ├── grail.yaml # project identity (placeholders for {name}/{root})
51
+ ├── endpoints.yaml # one inference endpoint, shows the base_url pattern
52
+ ├── llm.yaml # chat client + cache + concurrency
53
+ ├── embeddings.yaml # embedding client (reuses the same endpoint)
54
+ ├── indexing.yaml # chunking + extraction
55
+ ├── community.yaml # Leiden + incremental thresholds + report generation
56
+ ├── search.yaml # local + global search budgets
57
+ ├── storage.yaml # local backend with {root} placeholder
58
+ ├── prompts.yaml # custom prompt-pack search paths
59
+ └── vectorstore.yaml # LanceDB
60
+ ```
61
+
62
+ Every key is documented in [docs/glossary.md](../../docs/glossary.md).
63
+
64
+ ## Using a template
65
+
66
+ ```bash
67
+ grail init my-project --template low_cost_setup
68
+ ```
69
+
70
+ After scaffolding, fill in your API key in `.env` (or the project's `.env`),
71
+ edit `endpoints.yaml` if you want a different host, and you're ready to index.
72
+
73
+ ## Writing your own template
74
+
75
+ 1. Make a directory anywhere on disk, e.g. ``~/.grail/my-templates/acme/``.
76
+ 2. Copy any subset of the YAMLs above into it. ``grail.yaml`` is the minimum.
77
+ 3. Use placeholders ``{name}``, ``{root}``, ``{date}`` where you want them
78
+ substituted at init time. Other ``{…}`` sequences are passed through.
79
+ 4. Anything you don't include falls back to the defaults compiled into
80
+ ``grail/config.py``.
81
+
82
+ Use with:
83
+
84
+ ```bash
85
+ grail init my-project --template acme --templates-dir ~/.grail/my-templates
86
+ ```
87
+
88
+ ## Listing what's available
89
+
90
+ ```bash
91
+ grail init --list-templates # built-ins only
92
+ grail init --list-templates --templates-dir ~/.grail/ # plus your own
93
+ ```
94
+
95
+ ## Placeholders
96
+
97
+ | Placeholder | Replaced with |
98
+ |-------------|---------------------------------------------------------------|
99
+ | ``{name}`` | The ``--name`` flag (or project directory basename). |
100
+ | ``{root}`` | The absolute path of the new project directory. |
101
+ | ``{date}`` | ISO date at init time, e.g. ``2026-05-19``. |
@@ -0,0 +1,45 @@
1
+ # Community detection (Leiden) + report generation.
2
+ # See docs/incremental_pipeline.md for the algorithm and docs/glossary.md for keys.
3
+
4
+ # --- Hierarchical Leiden ---
5
+ # graspologic implementation. Larger max_cluster_size = fewer, broader communities.
6
+ max_cluster_size: 50
7
+ use_lcc: false # don't restrict to the largest connected component
8
+ strategy: leiden
9
+ seed: 42 # reproducible clustering — drop or change for variation
10
+
11
+ # --- Small-community merging (DBSCAN over centroids) ---
12
+ # Communities below min_community_size get merged with their nearest centroid.
13
+ min_community_size: 10
14
+ embedding_merge_eps: 0.5
15
+
16
+ # --- Incremental updates ---
17
+ # If (changed entities) / (total entities) >= threshold, re-cluster the affected
18
+ # subgraph with Leiden. Below threshold, label-propagate new nodes onto existing
19
+ # communities. 0.3 is the GraphRAG default; lower → more aggressive recompute.
20
+ incremental_change_threshold: 0.3
21
+
22
+ # --- Report generation (null = inherit from llm.yaml) ---
23
+ community_report_endpoint: null
24
+ community_report_model: null
25
+ json_corrector_endpoint: null
26
+ json_corrector_model: null
27
+
28
+ # Output token cap for community reports.
29
+ max_report_length: 4000
30
+
31
+ # Concurrency for report LLM calls. null = use global llm.concurrent_requests.
32
+ # report_concurrency: null
33
+
34
+ # Claims (covariates) are off by default in v0.1 — pipeline not wired yet.
35
+ include_covariates: false
36
+
37
+ # --- Report-level filtering ---
38
+ # Which Leiden hierarchy level to summarise. "coarsest" → smallest level number
39
+ # (fewest, broadest communities; default). Other values: "finest", "all", or an int.
40
+ community_level: coarsest
41
+
42
+ # Skip communities with fewer than this many entities at report-generation time.
43
+ # Defends against the long tail of singleton "communities" from isolated entities.
44
+ # 0 disables the filter.
45
+ min_report_size: 3
@@ -0,0 +1,29 @@
1
+ # Embedding client.
2
+ #
3
+ # Most providers serve chat AND embeddings under the same base_url — the OpenAI
4
+ # SDK POSTs to {base_url}/embeddings automatically. So reusing the `inference`
5
+ # endpoint here just works.
6
+ #
7
+ # If you ever need a different host for embeddings (separate server, different
8
+ # region, …) define a second endpoint in endpoints.yaml and reference it below.
9
+
10
+ endpoint: inference
11
+
12
+ # The embedding model name your endpoint accepts. Qwen3-Embedding-0.6B is a
13
+ # small, strong multilingual encoder available on DeepInfra.
14
+ #
15
+ # INVARIANT: this model must match between indexing and querying. Mismatched
16
+ # dimensions silently break recall.
17
+ model: Qwen/Qwen3-Embedding-0.6B
18
+
19
+ encoding_format: float
20
+
21
+ # Batching — most providers cap individual requests at a few hundred inputs.
22
+ # 256 is safe across DeepInfra / OpenAI / Together. Raise if your provider allows.
23
+ max_batch_size: 256
24
+ concurrent_requests: 8
25
+
26
+ request_timeout: 180.0
27
+ max_retries: 10
28
+ max_retry_wait: 10.0
29
+ sleep_on_rate_limit: 30.0