graphgrail 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graphgrail-0.1.0/.gitignore +62 -0
- graphgrail-0.1.0/LICENSE +21 -0
- graphgrail-0.1.0/PKG-INFO +68 -0
- graphgrail-0.1.0/configs/community.yaml +33 -0
- graphgrail-0.1.0/configs/embeddings.yaml +19 -0
- graphgrail-0.1.0/configs/endpoints.yaml +76 -0
- graphgrail-0.1.0/configs/grail.yaml +24 -0
- graphgrail-0.1.0/configs/indexing.yaml +62 -0
- graphgrail-0.1.0/configs/llm.yaml +24 -0
- graphgrail-0.1.0/configs/prompts.yaml +10 -0
- graphgrail-0.1.0/configs/reranker.yaml +33 -0
- graphgrail-0.1.0/configs/search.yaml +35 -0
- graphgrail-0.1.0/configs/storage.yaml +14 -0
- graphgrail-0.1.0/configs/templates/README.md +101 -0
- graphgrail-0.1.0/configs/templates/low_cost_setup/community.yaml +45 -0
- graphgrail-0.1.0/configs/templates/low_cost_setup/embeddings.yaml +29 -0
- graphgrail-0.1.0/configs/templates/low_cost_setup/endpoints.yaml +38 -0
- graphgrail-0.1.0/configs/templates/low_cost_setup/grail.yaml +18 -0
- graphgrail-0.1.0/configs/templates/low_cost_setup/indexing.yaml +85 -0
- graphgrail-0.1.0/configs/templates/low_cost_setup/llm.yaml +29 -0
- graphgrail-0.1.0/configs/templates/low_cost_setup/prompts.yaml +14 -0
- graphgrail-0.1.0/configs/templates/low_cost_setup/reranker.yaml +10 -0
- graphgrail-0.1.0/configs/templates/low_cost_setup/search.yaml +63 -0
- graphgrail-0.1.0/configs/templates/low_cost_setup/storage.yaml +14 -0
- graphgrail-0.1.0/configs/templates/low_cost_setup/vectorstore.yaml +19 -0
- graphgrail-0.1.0/configs/vectorstore.yaml +15 -0
- graphgrail-0.1.0/docs/benchmarks.md +156 -0
- graphgrail-0.1.0/docs/cli_chat.md +338 -0
- graphgrail-0.1.0/docs/comparison.md +323 -0
- graphgrail-0.1.0/docs/config.md +100 -0
- graphgrail-0.1.0/docs/getting-started.md +144 -0
- graphgrail-0.1.0/docs/glossary.md +232 -0
- graphgrail-0.1.0/docs/incremental_pipeline.md +544 -0
- graphgrail-0.1.0/docs/indexing.md +155 -0
- graphgrail-0.1.0/docs/llm.md +140 -0
- graphgrail-0.1.0/docs/model_selection/README.md +135 -0
- graphgrail-0.1.0/docs/model_selection/report.md +475 -0
- graphgrail-0.1.0/docs/preprocessing.md +145 -0
- graphgrail-0.1.0/docs/prompt_customization.md +865 -0
- graphgrail-0.1.0/docs/prompts.md +116 -0
- graphgrail-0.1.0/docs/python_api.md +370 -0
- graphgrail-0.1.0/docs/query.md +548 -0
- graphgrail-0.1.0/docs/reranker.md +121 -0
- graphgrail-0.1.0/docs/search_modes.md +698 -0
- graphgrail-0.1.0/docs/storage.md +123 -0
- graphgrail-0.1.0/docs/ui_frontend_analysis.md +176 -0
- graphgrail-0.1.0/docs/vectorstores.md +70 -0
- graphgrail-0.1.0/docs/viz.md +423 -0
- graphgrail-0.1.0/grail/__init__.py +59 -0
- graphgrail-0.1.0/grail/_version.py +1 -0
- graphgrail-0.1.0/grail/apps/__init__.py +0 -0
- graphgrail-0.1.0/grail/apps/chat/__init__.py +0 -0
- graphgrail-0.1.0/grail/apps/chat/auth.py +69 -0
- graphgrail-0.1.0/grail/apps/chat/database.py +349 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/dist/assets/index-2a40wZqj.css +1 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/dist/assets/index-DfRgtpKs.js +239 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/dist/index.html +17 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/index.html +16 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/package-lock.json +3990 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/package.json +29 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/App.tsx +76 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/components/ChatInput.tsx +131 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/components/ChatView.tsx +96 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/components/InfoPanel.tsx +119 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/components/Layout.tsx +118 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/components/LoginView.tsx +199 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/components/MarkdownRenderer.tsx +227 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/components/MessageBubble.tsx +237 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/components/ModeSelector.tsx +189 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/components/Sidebar.tsx +211 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/components/WelcomeView.tsx +182 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/index.css +166 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/lib/api.ts +162 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/lib/store.ts +395 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/main.tsx +10 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/src/vite-env.d.ts +1 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/tsconfig.app.json +21 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/tsconfig.app.tsbuildinfo +1 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/tsconfig.json +7 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/tsconfig.node.json +19 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/tsconfig.node.tsbuildinfo +1 -0
- graphgrail-0.1.0/grail/apps/chat/frontend/vite.config.ts +13 -0
- graphgrail-0.1.0/grail/apps/chat/schemas.py +90 -0
- graphgrail-0.1.0/grail/apps/chat/server.py +414 -0
- graphgrail-0.1.0/grail/apps/cli_chat/__init__.py +8 -0
- graphgrail-0.1.0/grail/apps/cli_chat/app.py +990 -0
- graphgrail-0.1.0/grail/apps/cli_chat/commands.py +181 -0
- graphgrail-0.1.0/grail/apps/cli_chat/reporter.py +73 -0
- graphgrail-0.1.0/grail/apps/cli_chat/widgets.py +496 -0
- graphgrail-0.1.0/grail/cli/__init__.py +7 -0
- graphgrail-0.1.0/grail/cli/banner.py +819 -0
- graphgrail-0.1.0/grail/cli/main.py +1663 -0
- graphgrail-0.1.0/grail/config.py +548 -0
- graphgrail-0.1.0/grail/core.py +1064 -0
- graphgrail-0.1.0/grail/export/__init__.py +5 -0
- graphgrail-0.1.0/grail/export/neo4j.py +298 -0
- graphgrail-0.1.0/grail/indexing/__init__.py +36 -0
- graphgrail-0.1.0/grail/indexing/communities.py +214 -0
- graphgrail-0.1.0/grail/indexing/community_reports.py +486 -0
- graphgrail-0.1.0/grail/indexing/entities_relationships.py +978 -0
- graphgrail-0.1.0/grail/indexing/entity_dedup.py +396 -0
- graphgrail-0.1.0/grail/indexing/incremental_community.py +382 -0
- graphgrail-0.1.0/grail/indexing/leiden.py +177 -0
- graphgrail-0.1.0/grail/indexing/loader.py +612 -0
- graphgrail-0.1.0/grail/indexing/preprocess.py +312 -0
- graphgrail-0.1.0/grail/indexing/run_manifest.py +274 -0
- graphgrail-0.1.0/grail/indexing/schema_migration.py +114 -0
- graphgrail-0.1.0/grail/indexing/stable_lcc.py +42 -0
- graphgrail-0.1.0/grail/indexing/summarize_descriptions.py +249 -0
- graphgrail-0.1.0/grail/llm/__init__.py +39 -0
- graphgrail-0.1.0/grail/llm/cache.py +122 -0
- graphgrail-0.1.0/grail/llm/cost.py +253 -0
- graphgrail-0.1.0/grail/llm/embeddings.py +228 -0
- graphgrail-0.1.0/grail/llm/providers.py +198 -0
- graphgrail-0.1.0/grail/llm/reranker.py +220 -0
- graphgrail-0.1.0/grail/llm/wrapper.py +654 -0
- graphgrail-0.1.0/grail/memory/__init__.py +55 -0
- graphgrail-0.1.0/grail/memory/_merge.py +288 -0
- graphgrail-0.1.0/grail/memory/analyses/__init__.py +70 -0
- graphgrail-0.1.0/grail/memory/analyses/alias_detect.py +190 -0
- graphgrail-0.1.0/grail/memory/analyses/edge_density.py +164 -0
- graphgrail-0.1.0/grail/memory/analyses/folder_split.py +135 -0
- graphgrail-0.1.0/grail/memory/analyses/membership.py +118 -0
- graphgrail-0.1.0/grail/memory/consolidate.py +164 -0
- graphgrail-0.1.0/grail/memory/identity.py +231 -0
- graphgrail-0.1.0/grail/memory/observation.py +165 -0
- graphgrail-0.1.0/grail/memory/project.py +1805 -0
- graphgrail-0.1.0/grail/memory/proposals.py +240 -0
- graphgrail-0.1.0/grail/memory/types.py +127 -0
- graphgrail-0.1.0/grail/prompts/__init__.py +25 -0
- graphgrail-0.1.0/grail/prompts/builtin/__init__.py +8 -0
- graphgrail-0.1.0/grail/prompts/builtin/claim_extraction.py +107 -0
- graphgrail-0.1.0/grail/prompts/builtin/community_report.py +170 -0
- graphgrail-0.1.0/grail/prompts/builtin/create_custom_entities.py +138 -0
- graphgrail-0.1.0/grail/prompts/builtin/entity_dedup.py +117 -0
- graphgrail-0.1.0/grail/prompts/builtin/entity_relation.py +267 -0
- graphgrail-0.1.0/grail/prompts/builtin/global_map.py +114 -0
- graphgrail-0.1.0/grail/prompts/builtin/global_reduce.py +77 -0
- graphgrail-0.1.0/grail/prompts/builtin/json_correction.py +83 -0
- graphgrail-0.1.0/grail/prompts/builtin/local_search.py +73 -0
- graphgrail-0.1.0/grail/prompts/builtin/summarize_description.py +105 -0
- graphgrail-0.1.0/grail/prompts/loader.py +172 -0
- graphgrail-0.1.0/grail/query/__init__.py +29 -0
- graphgrail-0.1.0/grail/query/agent.py +560 -0
- graphgrail-0.1.0/grail/query/cascade_search.py +376 -0
- graphgrail-0.1.0/grail/query/document_search.py +294 -0
- graphgrail-0.1.0/grail/query/global_search.py +287 -0
- graphgrail-0.1.0/grail/query/local_search.py +331 -0
- graphgrail-0.1.0/grail/query/recall_filter.py +312 -0
- graphgrail-0.1.0/grail/query/recall_search.py +264 -0
- graphgrail-0.1.0/grail/query/retrieval.py +425 -0
- graphgrail-0.1.0/grail/query/trace.py +119 -0
- graphgrail-0.1.0/grail/reporting/__init__.py +7 -0
- graphgrail-0.1.0/grail/reporting/rich_reporter.py +183 -0
- graphgrail-0.1.0/grail/schemas.py +382 -0
- graphgrail-0.1.0/grail/storage/__init__.py +29 -0
- graphgrail-0.1.0/grail/storage/base.py +127 -0
- graphgrail-0.1.0/grail/storage/local.py +116 -0
- graphgrail-0.1.0/grail/storage/s3.py +118 -0
- graphgrail-0.1.0/grail/utils/__init__.py +19 -0
- graphgrail-0.1.0/grail/utils/chunker.py +47 -0
- graphgrail-0.1.0/grail/utils/ids.py +19 -0
- graphgrail-0.1.0/grail/utils/text.py +44 -0
- graphgrail-0.1.0/grail/utils/tokens.py +29 -0
- graphgrail-0.1.0/grail/utils/zip.py +50 -0
- graphgrail-0.1.0/grail/vectorstores/__init__.py +29 -0
- graphgrail-0.1.0/grail/vectorstores/base.py +75 -0
- graphgrail-0.1.0/grail/vectorstores/chroma.py +145 -0
- graphgrail-0.1.0/grail/vectorstores/faiss.py +158 -0
- graphgrail-0.1.0/grail/vectorstores/lancedb.py +111 -0
- graphgrail-0.1.0/grail/viz/__init__.py +24 -0
- graphgrail-0.1.0/grail/viz/builder.py +129 -0
- graphgrail-0.1.0/grail/viz/colors.py +127 -0
- graphgrail-0.1.0/grail/viz/exporter.py +552 -0
- graphgrail-0.1.0/grail/viz/layout.py +315 -0
- graphgrail-0.1.0/grail/viz/template.py +1015 -0
- graphgrail-0.1.0/pyproject.toml +109 -0
- graphgrail-0.1.0/skills/grail/INSTALL.md +101 -0
- graphgrail-0.1.0/skills/grail/SKILL.md +149 -0
- graphgrail-0.1.0/skills/grail/agents/openai.yaml +20 -0
- graphgrail-0.1.0/skills/grail/assets/grail.kb.yaml.tpl +26 -0
- graphgrail-0.1.0/skills/grail/assets/grail.memory.yaml.tpl +44 -0
- graphgrail-0.1.0/skills/grail/assets/observation.md.tpl +14 -0
- graphgrail-0.1.0/skills/grail/references/config_reference.md +107 -0
- graphgrail-0.1.0/skills/grail/references/kb_mode.md +81 -0
- graphgrail-0.1.0/skills/grail/references/memory_mode.md +130 -0
- graphgrail-0.1.0/skills/grail/references/memory_tools.md +126 -0
- graphgrail-0.1.0/skills/grail/references/proposals.md +102 -0
- graphgrail-0.1.0/skills/grail/references/query_optimization.md +79 -0
- graphgrail-0.1.0/skills/grail/references/search_modes.md +94 -0
- graphgrail-0.1.0/skills/grail/references/troubleshooting.md +103 -0
- graphgrail-0.1.0/skills/grail/requirements.txt +9 -0
- graphgrail-0.1.0/skills/grail/scripts/_common.py +227 -0
- graphgrail-0.1.0/skills/grail/scripts/append.py +87 -0
- graphgrail-0.1.0/skills/grail/scripts/delete.py +52 -0
- graphgrail-0.1.0/skills/grail/scripts/edit.py +75 -0
- graphgrail-0.1.0/skills/grail/scripts/env_check.py +52 -0
- graphgrail-0.1.0/skills/grail/scripts/explore.py +94 -0
- graphgrail-0.1.0/skills/grail/scripts/index.py +83 -0
- graphgrail-0.1.0/skills/grail/scripts/init_project.py +108 -0
- graphgrail-0.1.0/skills/grail/scripts/list_grail_projects.py +53 -0
- graphgrail-0.1.0/skills/grail/scripts/memory/add_community.py +70 -0
- graphgrail-0.1.0/skills/grail/scripts/memory/add_entity.py +67 -0
- graphgrail-0.1.0/skills/grail/scripts/memory/add_observation.py +109 -0
- graphgrail-0.1.0/skills/grail/scripts/memory/add_relationship.py +62 -0
- graphgrail-0.1.0/skills/grail/scripts/memory/apply_proposal.py +56 -0
- graphgrail-0.1.0/skills/grail/scripts/memory/consolidate.py +54 -0
- graphgrail-0.1.0/skills/grail/scripts/memory/find_similar_entity.py +51 -0
- graphgrail-0.1.0/skills/grail/scripts/memory/list_proposals.py +49 -0
- graphgrail-0.1.0/skills/grail/scripts/memory/recall.py +76 -0
- graphgrail-0.1.0/skills/grail/scripts/query.py +151 -0
- graphgrail-0.1.0/skills/grail/scripts/setup.sh +24 -0
- graphgrail-0.1.0/skills/grail/scripts/status.py +97 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# --- Legacy/proprietary import baseline ---
|
|
2
|
+
# The _legacy_source/ tree is a verbatim copy of the proprietary `nirvana_agents/utilities/graphrag`
|
|
3
|
+
# snapshot used as the starting point for the GRAIL migration. It is intentionally kept in-tree
|
|
4
|
+
# so Claude Code sessions are self-contained (no path dependency on the proprietary monorepo).
|
|
5
|
+
# It must NEVER be committed to the public GRAIL repo. Delete the directory once migration is complete.
|
|
6
|
+
_legacy_source/
|
|
7
|
+
|
|
8
|
+
# --- Python ---
|
|
9
|
+
__pycache__/
|
|
10
|
+
*.py[cod]
|
|
11
|
+
*$py.class
|
|
12
|
+
*.so
|
|
13
|
+
.Python
|
|
14
|
+
*.egg
|
|
15
|
+
*.egg-info/
|
|
16
|
+
build/
|
|
17
|
+
/dist/
|
|
18
|
+
.eggs/
|
|
19
|
+
pip-wheel-metadata/
|
|
20
|
+
.pytest_cache/
|
|
21
|
+
.mypy_cache/
|
|
22
|
+
.ruff_cache/
|
|
23
|
+
.tox/
|
|
24
|
+
.coverage
|
|
25
|
+
.coverage.*
|
|
26
|
+
htmlcov/
|
|
27
|
+
.hypothesis/
|
|
28
|
+
|
|
29
|
+
# --- Virtual envs ---
|
|
30
|
+
.venv/
|
|
31
|
+
venv/
|
|
32
|
+
env/
|
|
33
|
+
ENV/
|
|
34
|
+
|
|
35
|
+
# --- Node (frontend builds) ---
|
|
36
|
+
node_modules/
|
|
37
|
+
|
|
38
|
+
# --- IDE / OS ---
|
|
39
|
+
.idea/
|
|
40
|
+
.vscode/
|
|
41
|
+
.DS_Store
|
|
42
|
+
Thumbs.db
|
|
43
|
+
|
|
44
|
+
# --- GRAIL runtime artifacts (default working dir) ---
|
|
45
|
+
# Outputs of indexing pipelines (parquet, lancedb, graphml). User overrides via config.
|
|
46
|
+
output/
|
|
47
|
+
cache/
|
|
48
|
+
lancedb/
|
|
49
|
+
*.parquet
|
|
50
|
+
*.graphml
|
|
51
|
+
mapping.json
|
|
52
|
+
|
|
53
|
+
# --- Notebook generated output ---
|
|
54
|
+
notebooks/_pyvis_graph.html
|
|
55
|
+
notebooks/.ipynb_checkpoints/
|
|
56
|
+
|
|
57
|
+
# --- Secrets / env ---
|
|
58
|
+
.env
|
|
59
|
+
.env.*
|
|
60
|
+
!.env.example
|
|
61
|
+
*.pem
|
|
62
|
+
*.key
|
graphgrail-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Cámara Chilena de Inteligencia Artificial
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: graphgrail
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Graph RAG with Advanced Integration and Learning — an open-source GraphRAG library with incremental updates, file-level provenance, and pluggable LLM/storage backends.
|
|
5
|
+
Project-URL: Repository, https://github.com/CAMARA-CHILENA-INTELIGENCIA-ARTIFICIAL/GRAIL
|
|
6
|
+
Author-email: Benjamin González Guerrero <ben@nirvana-ai.com>
|
|
7
|
+
License: Apache-2.0
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: graphrag,knowledge-graph,leiden,llm,rag,retrieval
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Requires-Dist: aiohttp>=3.9
|
|
20
|
+
Requires-Dist: aiosqlite>=0.20
|
|
21
|
+
Requires-Dist: graspologic>=3.4
|
|
22
|
+
Requires-Dist: httpx>=0.27
|
|
23
|
+
Requires-Dist: lancedb>=0.10
|
|
24
|
+
Requires-Dist: networkx>=3.2
|
|
25
|
+
Requires-Dist: numpy>=1.24
|
|
26
|
+
Requires-Dist: openai<2,>=1.40
|
|
27
|
+
Requires-Dist: pandas>=2.0
|
|
28
|
+
Requires-Dist: pyarrow>=14
|
|
29
|
+
Requires-Dist: pydantic<3,>=2.5
|
|
30
|
+
Requires-Dist: pypdf>=5.0
|
|
31
|
+
Requires-Dist: python-docx>=1.1
|
|
32
|
+
Requires-Dist: python-dotenv>=1.0
|
|
33
|
+
Requires-Dist: pyyaml>=6
|
|
34
|
+
Requires-Dist: rich>=13
|
|
35
|
+
Requires-Dist: scikit-learn>=1.3
|
|
36
|
+
Requires-Dist: scipy>=1.10
|
|
37
|
+
Requires-Dist: tenacity>=8.2
|
|
38
|
+
Requires-Dist: textual>=0.50
|
|
39
|
+
Requires-Dist: tiktoken>=0.7
|
|
40
|
+
Requires-Dist: typer>=0.12
|
|
41
|
+
Provides-Extra: chroma
|
|
42
|
+
Requires-Dist: chromadb>=0.5; extra == 'chroma'
|
|
43
|
+
Provides-Extra: dev
|
|
44
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
45
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
46
|
+
Requires-Dist: pytest-mock>=3.12; extra == 'dev'
|
|
47
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
48
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
49
|
+
Provides-Extra: faiss
|
|
50
|
+
Requires-Dist: faiss-cpu>=1.7; extra == 'faiss'
|
|
51
|
+
Provides-Extra: notebook
|
|
52
|
+
Requires-Dist: ipykernel>=6.29; extra == 'notebook'
|
|
53
|
+
Requires-Dist: ipywidgets>=8.1; extra == 'notebook'
|
|
54
|
+
Requires-Dist: jupyter>=1.0; extra == 'notebook'
|
|
55
|
+
Requires-Dist: matplotlib>=3.8; extra == 'notebook'
|
|
56
|
+
Requires-Dist: pyvis>=0.3.2; extra == 'notebook'
|
|
57
|
+
Provides-Extra: rerank
|
|
58
|
+
Requires-Dist: sentence-transformers>=2.7; extra == 'rerank'
|
|
59
|
+
Provides-Extra: s3
|
|
60
|
+
Requires-Dist: aioboto3>=12; extra == 's3'
|
|
61
|
+
Requires-Dist: boto3>=1.34; extra == 's3'
|
|
62
|
+
Provides-Extra: ui
|
|
63
|
+
Requires-Dist: bcrypt==4.0.1; extra == 'ui'
|
|
64
|
+
Requires-Dist: fastapi>=0.115; extra == 'ui'
|
|
65
|
+
Requires-Dist: passlib[bcrypt]>=1.7; extra == 'ui'
|
|
66
|
+
Requires-Dist: python-jose[cryptography]>=3.3; extra == 'ui'
|
|
67
|
+
Requires-Dist: sse-starlette>=2.0; extra == 'ui'
|
|
68
|
+
Requires-Dist: uvicorn[standard]>=0.30; extra == 'ui'
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Community detection + reporting config. See docs/community.md.
|
|
2
|
+
|
|
3
|
+
# Hierarchical Leiden (graspologic).
|
|
4
|
+
max_cluster_size: 50
|
|
5
|
+
use_lcc: false
|
|
6
|
+
strategy: leiden
|
|
7
|
+
seed: null # set an int for reproducible clustering
|
|
8
|
+
|
|
9
|
+
# Per-stage endpoint+model overrides. When null, inherit from configs/llm.yaml.
|
|
10
|
+
community_report_endpoint: null
|
|
11
|
+
community_report_model: null
|
|
12
|
+
json_corrector_endpoint: null
|
|
13
|
+
json_corrector_model: null
|
|
14
|
+
|
|
15
|
+
# JSON community report ceiling.
|
|
16
|
+
max_report_length: 8192
|
|
17
|
+
|
|
18
|
+
# Concurrency for community report LLM calls. null = use the global
|
|
19
|
+
# llm.concurrent_requests limit. Set lower to avoid overwhelming the
|
|
20
|
+
# endpoint with heavy report-generation calls.
|
|
21
|
+
# report_concurrency: null
|
|
22
|
+
|
|
23
|
+
# If true, include claims (covariates) in the report context.
|
|
24
|
+
include_covariates: false
|
|
25
|
+
|
|
26
|
+
# Incremental updates — change ratio threshold above which we re-cluster
|
|
27
|
+
# the affected subgraph instead of label-propagating new nodes.
|
|
28
|
+
incremental_change_threshold: 0.3
|
|
29
|
+
|
|
30
|
+
# After Leiden, small communities below this size get merged with their nearest
|
|
31
|
+
# centroid via DBSCAN over the entity embeddings.
|
|
32
|
+
min_community_size: 10
|
|
33
|
+
embedding_merge_eps: 0.5
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Embedding client config. See docs/llm.md.
|
|
2
|
+
#
|
|
3
|
+
# Same shape as configs/llm.yaml. endpoint references entries in
|
|
4
|
+
# configs/endpoints.yaml. The defaults below use DeepInfra's hosting of
|
|
5
|
+
# intfloat/multilingual-e5-large — a strong open-weights multilingual embedder.
|
|
6
|
+
|
|
7
|
+
endpoint: deepinfra
|
|
8
|
+
model: intfloat/multilingual-e5-large
|
|
9
|
+
encoding_format: float
|
|
10
|
+
|
|
11
|
+
# Batching / concurrency
|
|
12
|
+
max_batch_size: 1024
|
|
13
|
+
concurrent_requests: 30
|
|
14
|
+
|
|
15
|
+
# Timeouts / retries
|
|
16
|
+
request_timeout: 180.0
|
|
17
|
+
max_retries: 10
|
|
18
|
+
max_retry_wait: 10.0
|
|
19
|
+
sleep_on_rate_limit: 30.0
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Endpoint definitions. See docs/llm.md.
|
|
2
|
+
#
|
|
3
|
+
# Every entry here is a deployment of the OpenAI Chat Completions / Embeddings
|
|
4
|
+
# protocol. GRAIL is endpoint-agnostic by design — there are no vendor-specific
|
|
5
|
+
# code paths. To use a different vLLM cluster, sglang server, regional OpenAI
|
|
6
|
+
# proxy, or anything else, add an entry below and reference it by name in
|
|
7
|
+
# llm.yaml / embeddings.yaml / per-stage overrides.
|
|
8
|
+
#
|
|
9
|
+
# These defaults match the built-in registry in grail/llm/providers.py. User
|
|
10
|
+
# edits MERGE with the defaults (per-endpoint deep merge), so you only need to
|
|
11
|
+
# include entries you actually want to override.
|
|
12
|
+
|
|
13
|
+
openai:
|
|
14
|
+
base_url: https://api.openai.com/v1
|
|
15
|
+
api_key_env: OPENAI_API_KEY
|
|
16
|
+
|
|
17
|
+
anthropic:
|
|
18
|
+
base_url: https://api.anthropic.com/v1
|
|
19
|
+
api_key_env: ANTHROPIC_API_KEY
|
|
20
|
+
notes: Anthropic exposes /v1/messages over the OpenAI-compatible protocol.
|
|
21
|
+
|
|
22
|
+
deepinfra:
|
|
23
|
+
base_url: https://api.deepinfra.com/v1/openai
|
|
24
|
+
api_key_env: DEEPINFRA_API_KEY
|
|
25
|
+
|
|
26
|
+
together:
|
|
27
|
+
base_url: https://api.together.xyz/v1
|
|
28
|
+
api_key_env: TOGETHER_API_KEY
|
|
29
|
+
|
|
30
|
+
groq:
|
|
31
|
+
base_url: https://api.groq.com/openai/v1
|
|
32
|
+
api_key_env: GROQ_API_KEY
|
|
33
|
+
|
|
34
|
+
openrouter:
|
|
35
|
+
base_url: https://openrouter.ai/api/v1
|
|
36
|
+
api_key_env: OPENROUTER_API_KEY
|
|
37
|
+
|
|
38
|
+
# --- Self-hosted endpoints (no key by default) ---
|
|
39
|
+
|
|
40
|
+
ollama:
|
|
41
|
+
base_url: http://localhost:11434/v1
|
|
42
|
+
api_key_env: OLLAMA_API_KEY
|
|
43
|
+
requires_key: false
|
|
44
|
+
notes: Local Ollama server.
|
|
45
|
+
|
|
46
|
+
vllm:
|
|
47
|
+
base_url: http://localhost:8000/v1
|
|
48
|
+
api_key_env: VLLM_API_KEY
|
|
49
|
+
requires_key: false
|
|
50
|
+
notes: Self-hosted vLLM server speaking OpenAI's protocol.
|
|
51
|
+
|
|
52
|
+
sglang:
|
|
53
|
+
base_url: http://localhost:30000/v1
|
|
54
|
+
api_key_env: SGLANG_API_KEY
|
|
55
|
+
requires_key: false
|
|
56
|
+
notes: Self-hosted SGLang server.
|
|
57
|
+
|
|
58
|
+
lmstudio:
|
|
59
|
+
base_url: http://localhost:1234/v1
|
|
60
|
+
api_key_env: LMSTUDIO_API_KEY
|
|
61
|
+
requires_key: false
|
|
62
|
+
notes: LM Studio local server.
|
|
63
|
+
|
|
64
|
+
local:
|
|
65
|
+
base_url: http://localhost:8000/v1
|
|
66
|
+
api_key_env: LOCAL_API_KEY
|
|
67
|
+
requires_key: false
|
|
68
|
+
notes: Catch-all entry for arbitrary local servers.
|
|
69
|
+
|
|
70
|
+
# --- Add your own deployments below ---
|
|
71
|
+
#
|
|
72
|
+
# my-internal-vllm:
|
|
73
|
+
# base_url: https://vllm.internal.corp/v1
|
|
74
|
+
# api_key_env: INTERNAL_VLLM_KEY
|
|
75
|
+
# requires_key: true
|
|
76
|
+
# notes: Behind the corporate VPN.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# GRAIL master config.
|
|
2
|
+
# Loaded by `grail.config.load_config(path)`. Sibling per-module YAMLs in this
|
|
3
|
+
# directory are auto-merged into the matching section, so you can split overrides
|
|
4
|
+
# by concern. Documentation for every key lives in docs/<module>.md.
|
|
5
|
+
|
|
6
|
+
project_name: example
|
|
7
|
+
# Resolves with ~ expansion. All on-disk artefacts (parquets, lancedb, cache) live here.
|
|
8
|
+
root_dir: ~/.grail/projects/example
|
|
9
|
+
|
|
10
|
+
# Each block below mirrors a Pydantic submodel in grail/config.py. Anything you omit
|
|
11
|
+
# falls back to the default in code. To split this file by module, leave the section
|
|
12
|
+
# empty here and drop e.g. configs/llm.yaml next to this one — both load and merge.
|
|
13
|
+
|
|
14
|
+
llm: {}
|
|
15
|
+
embeddings: {}
|
|
16
|
+
indexing: {}
|
|
17
|
+
community: {}
|
|
18
|
+
search: {}
|
|
19
|
+
storage:
|
|
20
|
+
backend: local
|
|
21
|
+
# If unset, defaults to {root_dir}.
|
|
22
|
+
root: ~/.grail/projects/example
|
|
23
|
+
prompts: {}
|
|
24
|
+
vectorstore: {}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Indexing pipeline config. See docs/indexing.md.
|
|
2
|
+
|
|
3
|
+
# Chunking
|
|
4
|
+
chunk_size: 2000
|
|
5
|
+
chunk_overlap: 50
|
|
6
|
+
encoding_name: cl100k_base
|
|
7
|
+
document_boundary: "\n\n---DOCUMENT_BOUNDARY---\n\n"
|
|
8
|
+
|
|
9
|
+
# Layout under storage root.
|
|
10
|
+
input_folder: input
|
|
11
|
+
output_folder: output
|
|
12
|
+
cache_folder: cache
|
|
13
|
+
|
|
14
|
+
# Per-stage endpoint+model overrides. When null, inherit from configs/llm.yaml.
|
|
15
|
+
# Set one or both per stage — e.g. cheap extraction on Groq, headline summarization on OpenAI.
|
|
16
|
+
entity_relation_endpoint: null
|
|
17
|
+
entity_relation_model: null
|
|
18
|
+
summarization_endpoint: null
|
|
19
|
+
summarization_model: null
|
|
20
|
+
|
|
21
|
+
# Entity types normalized to UPPER_SNAKE_CASE. Minimum 5 required.
|
|
22
|
+
# Always include PERSON and ORGANIZATION as baseline types.
|
|
23
|
+
# Add domain-specific types, or use discover_entity_types to let the LLM propose them.
|
|
24
|
+
entity_types:
|
|
25
|
+
- person
|
|
26
|
+
- organization
|
|
27
|
+
- location
|
|
28
|
+
- event
|
|
29
|
+
- concept
|
|
30
|
+
|
|
31
|
+
# When true, the LLM reads a corpus sample before extraction and proposes
|
|
32
|
+
# additional entity types (merged with the list above, capped at max_entity_types).
|
|
33
|
+
# Also available as CLI flag: grail index --discover-entities
|
|
34
|
+
# discover_entity_types: false
|
|
35
|
+
# max_entity_types: 15
|
|
36
|
+
|
|
37
|
+
# Extraction delimiters — the entity/relationship parser is contract-bound to
|
|
38
|
+
# these tokens. Change them only if you have a custom entity_relation prompt
|
|
39
|
+
# that uses different delimiters. When absent, the built-in defaults apply.
|
|
40
|
+
# tuple_delimiter: "<|>"
|
|
41
|
+
# record_delimiter: "##"
|
|
42
|
+
# completion_delimiter: "<|COMPLETE|>"
|
|
43
|
+
# start_delimiter: "<|START_OUTPUT|>"
|
|
44
|
+
|
|
45
|
+
# --- Token budgets per LLM call ---
|
|
46
|
+
# extraction_max_tokens: max output tokens for each entity/relationship extraction call.
|
|
47
|
+
# entity_discovery_max_tokens: max output tokens for the create-entities discovery call.
|
|
48
|
+
# max_summarization_tokens: max tokens for entity description summarization.
|
|
49
|
+
extraction_max_tokens: 8192
|
|
50
|
+
entity_discovery_max_tokens: 2048
|
|
51
|
+
max_summarization_tokens: 756
|
|
52
|
+
|
|
53
|
+
# --- Per-stage concurrency ---
|
|
54
|
+
# Override the global llm.concurrent_requests for specific stages.
|
|
55
|
+
# null = use the global limit. Useful to run more extraction calls in parallel
|
|
56
|
+
# (cheap, fast) while throttling heavier community-report generation.
|
|
57
|
+
# extraction_concurrency: null
|
|
58
|
+
# summarization_concurrency: null
|
|
59
|
+
|
|
60
|
+
# Gleanings: the upstream graphrag setting that re-asks the LLM to find missed
|
|
61
|
+
# entities. 0 = disabled. Costs roughly N× more tokens per chunk.
|
|
62
|
+
max_gleanings: 0
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# LLM client config. See docs/llm.md.
|
|
2
|
+
#
|
|
3
|
+
# Endpoint and model are separate. `endpoint` is a name referenced in
|
|
4
|
+
# configs/endpoints.yaml (defaults cover openai, anthropic, deepinfra, together,
|
|
5
|
+
# groq, openrouter, ollama, vllm, sglang, lmstudio, local).
|
|
6
|
+
|
|
7
|
+
# Which deployment of the OpenAI protocol to use by default.
|
|
8
|
+
endpoint: openai
|
|
9
|
+
# Model name within that endpoint. Whatever string the endpoint accepts.
|
|
10
|
+
model: gpt-4o-mini
|
|
11
|
+
|
|
12
|
+
# Concurrency / timeouts
|
|
13
|
+
concurrent_requests: 15
|
|
14
|
+
request_timeout: 180.0
|
|
15
|
+
max_retries: 10
|
|
16
|
+
max_retry_wait: 10.0
|
|
17
|
+
sleep_on_rate_limit: 30.0
|
|
18
|
+
|
|
19
|
+
debug: false
|
|
20
|
+
|
|
21
|
+
# Disk cache. When enabled, identical (endpoint, model, messages, params) tuples
|
|
22
|
+
# are reused. Useful during iteration.
|
|
23
|
+
cache_enabled: false
|
|
24
|
+
cache_dir: null # null → {root_dir}/cache/llm
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Prompts config. See docs/prompts.md.
|
|
2
|
+
|
|
3
|
+
# Extra directories searched (in order) before grail/prompts/builtin/. Each file
|
|
4
|
+
# in these directories must expose NAME, REQUIRED_PARAMS, build_messages(**params).
|
|
5
|
+
custom_paths: []
|
|
6
|
+
|
|
7
|
+
# strict=true requires that the custom directory provide every built-in prompt
|
|
8
|
+
# (entity_relation, summarize_description, community_report, json_correction,
|
|
9
|
+
# create_custom_entities, local_search, global_map, global_reduce, claim_extraction).
|
|
10
|
+
strict: false
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Reranker config (optional). See docs/reranker.md.
|
|
2
|
+
#
|
|
3
|
+
# When enabled, a cross-encoder re-ranker scores entity and text-unit candidates
|
|
4
|
+
# after the initial vector similarity retrieval. This improves which items fill
|
|
5
|
+
# the limited context window sent to the LLM, at the cost of one extra API call.
|
|
6
|
+
#
|
|
7
|
+
# Toggle per-query with --rerank / --no-rerank on the CLI, or use_reranker= in
|
|
8
|
+
# the Python API, without changing this config.
|
|
9
|
+
|
|
10
|
+
enabled: false
|
|
11
|
+
|
|
12
|
+
# Endpoint name (from endpoints.yaml) that hosts the reranker model.
|
|
13
|
+
# The reranker uses a dedicated HTTP API (not OpenAI Chat Completions).
|
|
14
|
+
endpoint: deepinfra
|
|
15
|
+
|
|
16
|
+
# Cross-encoder model name. Passed to the inference API.
|
|
17
|
+
model: Qwen/Qwen3-Reranker-0.6B
|
|
18
|
+
|
|
19
|
+
# Full URL override. When null, auto-derived from endpoint name.
|
|
20
|
+
# DeepInfra: https://api.deepinfra.com/v1/inference/{model}
|
|
21
|
+
base_url: null
|
|
22
|
+
|
|
23
|
+
# Over-fetch factor: retrieve top_k * overfetch_factor entities by vector
|
|
24
|
+
# similarity, then rerank and trim to top_k. Higher = more candidates for the
|
|
25
|
+
# reranker to choose from, but more tokens sent to the reranker API.
|
|
26
|
+
overfetch_factor: 3
|
|
27
|
+
|
|
28
|
+
# Which retrieval stages to rerank.
|
|
29
|
+
rerank_entities: true
|
|
30
|
+
rerank_text_units: true
|
|
31
|
+
|
|
32
|
+
# HTTP timeout for the reranker API call (seconds).
|
|
33
|
+
request_timeout: 30.0
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Search config. See docs/query.md.
|
|
2
|
+
|
|
3
|
+
# Per-stage endpoint+model overrides. When null, inherit from configs/llm.yaml.
|
|
4
|
+
local_search_endpoint: null
|
|
5
|
+
local_search_model: null
|
|
6
|
+
global_search_endpoint: null
|
|
7
|
+
global_search_model: null
|
|
8
|
+
|
|
9
|
+
# --- Local search budgeting ---
|
|
10
|
+
local_max_tokens: 12000
|
|
11
|
+
local_text_unit_prop: 0.5
|
|
12
|
+
local_community_prop: 0.1
|
|
13
|
+
local_conversation_history_max_turns: 5
|
|
14
|
+
local_top_k_entities: 10
|
|
15
|
+
local_top_k_relationships: 10
|
|
16
|
+
|
|
17
|
+
# When false (default), community reports include full LLM-generated content
|
|
18
|
+
# with detailed findings. When true, only the one-line summary is used (legacy
|
|
19
|
+
# behaviour for tight token budgets or small-context models).
|
|
20
|
+
use_community_summary: false
|
|
21
|
+
|
|
22
|
+
# --- Global search budgeting ---
|
|
23
|
+
global_map_max_tokens: 2000
|
|
24
|
+
global_reduce_max_tokens: 8192
|
|
25
|
+
# When the assembled community-report context exceeds this token budget, GRAIL
|
|
26
|
+
# falls back to a map-reduce path. Otherwise it does a single reduce pass.
|
|
27
|
+
global_chunk_size: 100000
|
|
28
|
+
global_concurrency: 5
|
|
29
|
+
|
|
30
|
+
# Maximum tokens the LLM can generate in its response.
|
|
31
|
+
response_max_tokens: 16384
|
|
32
|
+
|
|
33
|
+
# Description of the desired response style — injected into the global reduce
|
|
34
|
+
# prompt's artifact_instructions block.
|
|
35
|
+
response_type: "Multiple Paragraphs"
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Storage backend config. See docs/storage.md.
|
|
2
|
+
|
|
3
|
+
# "local" (default) writes everything under `root` on the filesystem.
|
|
4
|
+
# "s3" writes objects under `s3_bucket/s3_prefix` (requires the [s3] extra).
|
|
5
|
+
backend: local
|
|
6
|
+
|
|
7
|
+
# Local backend
|
|
8
|
+
root: ~/.grail/projects/default
|
|
9
|
+
|
|
10
|
+
# S3 backend (only used when backend=s3)
|
|
11
|
+
s3_bucket: null
|
|
12
|
+
s3_prefix: null
|
|
13
|
+
s3_region: null
|
|
14
|
+
s3_endpoint_url: null # set for MinIO / S3-compatible stores
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# GRAIL configuration templates
|
|
2
|
+
|
|
3
|
+
A **template** is a directory of YAML files matching the GRAIL config layout
|
|
4
|
+
(``grail.yaml`` + optional ``endpoints.yaml``, ``llm.yaml``, ``embeddings.yaml``,
|
|
5
|
+
``indexing.yaml``, ``community.yaml``, ``search.yaml``, ``storage.yaml``,
|
|
6
|
+
``prompts.yaml``, ``vectorstore.yaml``).
|
|
7
|
+
|
|
8
|
+
When you scaffold a project with ``grail init <project_dir> --template NAME``,
|
|
9
|
+
every recognised YAML in that template directory is copied into the new project
|
|
10
|
+
with ``{name}``, ``{root}``, and ``{date}`` placeholders substituted.
|
|
11
|
+
|
|
12
|
+
## The "everything is OpenAI API" position
|
|
13
|
+
|
|
14
|
+
GRAIL only speaks the OpenAI Chat Completions + Embeddings protocol. An
|
|
15
|
+
**endpoint** is just one deployment of that protocol — a base URL the SDK posts
|
|
16
|
+
to. Any compatible host plugs in: OpenAI itself, DeepInfra, Together, Groq,
|
|
17
|
+
your own vLLM / SGLang / LM Studio server.
|
|
18
|
+
|
|
19
|
+
The base URL is the ``/v1`` prefix. The SDK appends ``/chat/completions`` or
|
|
20
|
+
``/embeddings`` automatically. **Do not include ``/chat/completions`` or
|
|
21
|
+
``/embeddings`` in ``base_url``.**
|
|
22
|
+
|
|
23
|
+
Examples of valid ``base_url`` values:
|
|
24
|
+
|
|
25
|
+
| Host | base_url |
|
|
26
|
+
|-----------------------|------------------------------------------------|
|
|
27
|
+
| OpenAI | ``https://api.openai.com/v1`` |
|
|
28
|
+
| DeepInfra | ``https://api.deepinfra.com/v1/openai`` |
|
|
29
|
+
| Together | ``https://api.together.xyz/v1`` |
|
|
30
|
+
| Groq | ``https://api.groq.com/openai/v1`` |
|
|
31
|
+
| Local vLLM | ``http://localhost:8000/v1`` |
|
|
32
|
+
| Local SGLang | ``http://localhost:30000/v1`` |
|
|
33
|
+
|
|
34
|
+
## Built-in templates
|
|
35
|
+
|
|
36
|
+
### `low_cost_setup` — the one we ship
|
|
37
|
+
|
|
38
|
+
A fully-filled GRAIL project biased for cost:
|
|
39
|
+
|
|
40
|
+
- One inference endpoint pointed at DeepInfra (swap the ``base_url`` to retarget).
|
|
41
|
+
- Conservative concurrency (8 in-flight calls).
|
|
42
|
+
- LLM disk cache **on** so re-running indexing is free.
|
|
43
|
+
- Single-pass extraction (``max_gleanings: 0``).
|
|
44
|
+
- Tighter search token budgets than the GRAIL defaults.
|
|
45
|
+
|
|
46
|
+
Files included:
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
configs/templates/low_cost_setup/
|
|
50
|
+
├── grail.yaml # project identity (placeholders for {name}/{root})
|
|
51
|
+
├── endpoints.yaml # one inference endpoint, shows the base_url pattern
|
|
52
|
+
├── llm.yaml # chat client + cache + concurrency
|
|
53
|
+
├── embeddings.yaml # embedding client (reuses the same endpoint)
|
|
54
|
+
├── indexing.yaml # chunking + extraction
|
|
55
|
+
├── community.yaml # Leiden + incremental thresholds + report generation
|
|
56
|
+
├── search.yaml # local + global search budgets
|
|
57
|
+
├── storage.yaml # local backend with {root} placeholder
|
|
58
|
+
├── prompts.yaml # custom prompt-pack search paths
|
|
59
|
+
└── vectorstore.yaml # LanceDB
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Every key is documented in [docs/glossary.md](../../docs/glossary.md).
|
|
63
|
+
|
|
64
|
+
## Using a template
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
grail init my-project --template low_cost_setup
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
After scaffolding, fill in your API key in `.env` (or the project's `.env`),
|
|
71
|
+
edit `endpoints.yaml` if you want a different host, and you're ready to index.
|
|
72
|
+
|
|
73
|
+
## Writing your own template
|
|
74
|
+
|
|
75
|
+
1. Make a directory anywhere on disk, e.g. ``~/.grail/my-templates/acme/``.
|
|
76
|
+
2. Copy any subset of the YAMLs above into it. ``grail.yaml`` is the minimum.
|
|
77
|
+
3. Use placeholders ``{name}``, ``{root}``, ``{date}`` where you want them
|
|
78
|
+
substituted at init time. Other ``{…}`` sequences are passed through.
|
|
79
|
+
4. Anything you don't include falls back to the defaults compiled into
|
|
80
|
+
``grail/config.py``.
|
|
81
|
+
|
|
82
|
+
Use with:
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
grail init my-project --template acme --templates-dir ~/.grail/my-templates
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Listing what's available
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
grail init --list-templates # built-ins only
|
|
92
|
+
grail init --list-templates --templates-dir ~/.grail/ # plus your own
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Placeholders
|
|
96
|
+
|
|
97
|
+
| Placeholder | Replaced with |
|
|
98
|
+
|-------------|---------------------------------------------------------------|
|
|
99
|
+
| ``{name}`` | The ``--name`` flag (or project directory basename). |
|
|
100
|
+
| ``{root}`` | The absolute path of the new project directory. |
|
|
101
|
+
| ``{date}`` | ISO date at init time, e.g. ``2026-05-19``. |
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Community detection (Leiden) + report generation.
|
|
2
|
+
# See docs/incremental_pipeline.md for the algorithm and docs/glossary.md for keys.
|
|
3
|
+
|
|
4
|
+
# --- Hierarchical Leiden ---
|
|
5
|
+
# graspologic implementation. Larger max_cluster_size = fewer, broader communities.
|
|
6
|
+
max_cluster_size: 50
|
|
7
|
+
use_lcc: false # don't restrict to the largest connected component
|
|
8
|
+
strategy: leiden
|
|
9
|
+
seed: 42 # reproducible clustering — drop or change for variation
|
|
10
|
+
|
|
11
|
+
# --- Small-community merging (DBSCAN over centroids) ---
|
|
12
|
+
# Communities below min_community_size get merged with their nearest centroid.
|
|
13
|
+
min_community_size: 10
|
|
14
|
+
embedding_merge_eps: 0.5
|
|
15
|
+
|
|
16
|
+
# --- Incremental updates ---
|
|
17
|
+
# If (changed entities) / (total entities) >= threshold, re-cluster the affected
|
|
18
|
+
# subgraph with Leiden. Below threshold, label-propagate new nodes onto existing
|
|
19
|
+
# communities. 0.3 is the GraphRAG default; lower → more aggressive recompute.
|
|
20
|
+
incremental_change_threshold: 0.3
|
|
21
|
+
|
|
22
|
+
# --- Report generation (null = inherit from llm.yaml) ---
|
|
23
|
+
community_report_endpoint: null
|
|
24
|
+
community_report_model: null
|
|
25
|
+
json_corrector_endpoint: null
|
|
26
|
+
json_corrector_model: null
|
|
27
|
+
|
|
28
|
+
# Output token cap for community reports.
|
|
29
|
+
max_report_length: 4000
|
|
30
|
+
|
|
31
|
+
# Concurrency for report LLM calls. null = use global llm.concurrent_requests.
|
|
32
|
+
# report_concurrency: null
|
|
33
|
+
|
|
34
|
+
# Claims (covariates) are off by default in v0.1 — pipeline not wired yet.
|
|
35
|
+
include_covariates: false
|
|
36
|
+
|
|
37
|
+
# --- Report-level filtering ---
|
|
38
|
+
# Which Leiden hierarchy level to summarise. "coarsest" → smallest level number
|
|
39
|
+
# (fewest, broadest communities; default). Other values: "finest", "all", or an int.
|
|
40
|
+
community_level: coarsest
|
|
41
|
+
|
|
42
|
+
# Skip communities with fewer than this many entities at report-generation time.
|
|
43
|
+
# Defends against the long tail of singleton "communities" from isolated entities.
|
|
44
|
+
# 0 disables the filter.
|
|
45
|
+
min_report_size: 3
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Embedding client.
|
|
2
|
+
#
|
|
3
|
+
# Most providers serve chat AND embeddings under the same base_url — the OpenAI
|
|
4
|
+
# SDK POSTs to {base_url}/embeddings automatically. So reusing the `inference`
|
|
5
|
+
# endpoint here just works.
|
|
6
|
+
#
|
|
7
|
+
# If you ever need a different host for embeddings (separate server, different
|
|
8
|
+
# region, …) define a second endpoint in endpoints.yaml and reference it below.
|
|
9
|
+
|
|
10
|
+
endpoint: inference
|
|
11
|
+
|
|
12
|
+
# The embedding model name your endpoint accepts. Qwen3-Embedding-0.6B is a
|
|
13
|
+
# small, strong multilingual encoder available on DeepInfra.
|
|
14
|
+
#
|
|
15
|
+
# INVARIANT: this model must match between indexing and querying. Mismatched
|
|
16
|
+
# dimensions silently break recall.
|
|
17
|
+
model: Qwen/Qwen3-Embedding-0.6B
|
|
18
|
+
|
|
19
|
+
encoding_format: float
|
|
20
|
+
|
|
21
|
+
# Batching — most providers cap individual requests at a few hundred inputs.
|
|
22
|
+
# 256 is safe across DeepInfra / OpenAI / Together. Raise if your provider allows.
|
|
23
|
+
max_batch_size: 256
|
|
24
|
+
concurrent_requests: 8
|
|
25
|
+
|
|
26
|
+
request_timeout: 180.0
|
|
27
|
+
max_retries: 10
|
|
28
|
+
max_retry_wait: 10.0
|
|
29
|
+
sleep_on_rate_limit: 30.0
|