ragx-cli 0.9.0__tar.gz → 0.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/CHANGELOG.md +23 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/CLAUDE.md +13 -9
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/PKG-INFO +23 -13
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/README.md +22 -12
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/pyproject.toml +1 -1
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/skills/ragx-cli/SKILL.md +5 -2
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/cli/pipeline.py +15 -1
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/graph_export.py +5 -11
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/indexer.py +19 -9
- ragx_cli-0.10.0/src/ragx/core/reedge.py +106 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/store.py +8 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/js/layout.js +1 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/js/main.js +4 -1
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_cli.py +14 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_graph_export.py +13 -12
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_integration.py +1 -0
- ragx_cli-0.10.0/tests/test_reedge.py +127 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/SKILL.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/api-cheatsheet.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/collection-management/python.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/collection-management/typescript.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/configuration.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/data-model.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/data-operations/python.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/data-operations/typescript.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/embedding.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/hybrid-search/python.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/hybrid-search/typescript.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/multimodal-search/python.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/multimodal-search/typescript.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/quick-start/python.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/quick-start/typescript.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/rag-system/python.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/rag-system/typescript.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/reranker.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/troubleshooting.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/vector-search/python.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.agents/skills/zvec/vector-search/typescript.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.github/workflows/publish.yml +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/.gitignore +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/AGENTS.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/CONTRACTS-PHASE23.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/CONTRACTS.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/LICENSE +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/docs/feature-temporal-weighting.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/docs/proposal-models-command-interactive-flow.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/ragx-cli-plan.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/research/bge-m3-colbert-multi-vector-late-interaction-for-ragx-rerank-alternative-and-token-level-graph-edges.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/research/bge-m3-dense-embeddings-as-ragx-provider-multilingual-quality-and-threshold-calibration.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/research/bge-m3-dense-q8-vs-nomic-q4-benchmark-2026-07-12-worktree-eval-results.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/research/bge-m3-sparse-lexical-weights-hybrid-retrieval-leg-for-ragx-vs-sqlite-fts5-bm25.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/research/bge-m3-subchunk-edge-ablation-2026-07-13-ties-again-and-beta-zero-shortlist-mechanism.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/research/fine-grained-sub-chunk-edges-with-coarse-chunk-nodes-multi-granularity-graph-rag-literature-validation.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/research/lm-studio-model-download-api-and-curated-embedding-reranker-catalog-for-ragx-models-command.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/research/subchunk-edge-ablation-2026-07-12-two-round-experiment-protocol-results-and-verdict.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/research/subchunk-size-sweep-2026-10-21-default-edge-source-subchunk-size-choice.md +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/skills-lock.json +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/__init__.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/cli/__init__.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/cli/app.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/cli/detect.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/cli/doctor_cmd.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/cli/eval_cmd.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/cli/init_prompt.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/cli/inspect_cmd.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/cli/models_cmd.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/cli/models_ui.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/cli/output.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/cli/web_cmd.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/__init__.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/catalog.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/chunking.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/communities.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/config.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/context.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/corpus_globs.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/discovery.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/doctor.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/errors.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/eval.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/expansion.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/fusion.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/graph.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/grouping.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/lmstudio.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/models.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/query.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/scoring.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/traversal.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/core/vectors.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/providers/__init__.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/providers/base.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/providers/llama_embedder.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/providers/llama_process.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/providers/llama_server.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/providers/lmstudio_process.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/providers/openai_compat.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/providers/registry.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/providers/st_reranker.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/__init__.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/handlers.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/query_runner.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/server.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/index.html +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/js/colors.js +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/js/graph.js +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/js/query.js +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/js/render.js +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/js/util.js +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/styles.css +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/vendor/LICENSE-graphology-library.txt +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/vendor/LICENSE-graphology.txt +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/vendor/LICENSE-sigma.txt +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/vendor/graphology-library.min.js +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/vendor/graphology.umd.min.js +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static/vendor/sigma.min.js +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/src/ragx/web/static_files.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/__init__.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/conftest.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_chunking.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_communities.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_config_rc.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_context.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_discovery.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_doctor.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_eval.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_expansion.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_fusion.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_graph.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_grouping.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_inspect.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_llama_server.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_lmstudio_autostart.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_models.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_providers.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_query_cmd.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_store.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_traversal.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_vectors.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_web_cmd.py +0 -0
- {ragx_cli-0.9.0 → ragx_cli-0.10.0}/tests/test_web_server.py +0 -0
|
@@ -4,6 +4,29 @@ All notable changes to `ragx-cli` are documented here. The format follows
|
|
|
4
4
|
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions follow
|
|
5
5
|
[Semantic Versioning](https://semver.org/).
|
|
6
6
|
|
|
7
|
+
## [0.10.0] — 2026-10-21
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- `ragx-cli index --edges-only` — rebuilds sub-chunks, graph edges and communities from the
|
|
12
|
+
stored chunks per the current `[graph]` config, without re-chunking or re-embedding files.
|
|
13
|
+
Lets an existing index switch `graph.edge_source` (or `subchunk_size_tokens`) cheaply: chunk
|
|
14
|
+
embeddings and the vector index are kept; a chunk that stays whole reuses its own stored
|
|
15
|
+
vector, so only the sub-chunk pieces a split chunk actually produces go through the embedder.
|
|
16
|
+
Mutually exclusive with `--full`/`--changed`. The incremental-index manifest mismatch message
|
|
17
|
+
for `edge_source`/`subchunk_size_tokens` now also suggests `--edges-only`.
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
|
|
21
|
+
- `ragx-cli web` no longer requires an index built with `graph.edge_source = "subchunk"`: it
|
|
22
|
+
renders whatever edges the index has — sub-chunk similarity when the index has them, otherwise
|
|
23
|
+
the stored whole-chunk cosine edges. An index with zero edges renders as a node-only cloud.
|
|
24
|
+
|
|
25
|
+
### Fixed
|
|
26
|
+
|
|
27
|
+
- `ragx-cli web` layout: ForceAtlas2 now uses edge weights (`edgeWeightInfluence = 1`), so more
|
|
28
|
+
similar chunks sit closer together. Before, every edge pulled equally.
|
|
29
|
+
|
|
7
30
|
## [0.9.0] — 2026-10-21
|
|
8
31
|
|
|
9
32
|
### Added
|
|
@@ -73,8 +73,10 @@ src/ragx/
|
|
|
73
73
|
catalog.py # curated embedding/reranker catalog (tiers, prefixes) + spec detect
|
|
74
74
|
lmstudio.py # `lms` CLI wrapper: find/download (`lms get --yes`)/`lms ls --json`;
|
|
75
75
|
# LM Studio can't download plain safetensors — rerankers stay on HF
|
|
76
|
-
graph_export.py # build_graph: ragx.graph.v1 payload for `web`;
|
|
77
|
-
# index
|
|
76
|
+
graph_export.py # build_graph: ragx.graph.v1 payload for `web`; renders whatever edges the
|
|
77
|
+
# index has, subchunk or chunk-mode cosine (see Flows)
|
|
78
|
+
reedge.py # rebuild_edges: `index --edges-only` — rebuilds sub-chunks/edges/
|
|
79
|
+
# communities from stored chunks, no re-chunk/re-embed (see Flows)
|
|
78
80
|
providers/ # base.py protocols (Embedder/Generator/Reranker); openai_compat.py;
|
|
79
81
|
# st_reranker.py; lmstudio_process.py (LM Studio autostart);
|
|
80
82
|
# llama_process.py (managed llama-server lifecycle) +
|
|
@@ -88,7 +90,7 @@ src/ragx/
|
|
|
88
90
|
# handlers.py (routing), query_runner.py (lazy providers, lock-serialized
|
|
89
91
|
# queries), static_files.py (path-traversal-safe static serving);
|
|
90
92
|
# static/ (frontend-owned: index.html, JS/CSS, vendored sigma.js)
|
|
91
|
-
tests/ #
|
|
93
|
+
tests/ # 301 tests; mocked HTTP (respx), FakeEmbedder integration tests, no live network
|
|
92
94
|
```
|
|
93
95
|
|
|
94
96
|
## Flows
|
|
@@ -108,8 +110,10 @@ max cos over sub-chunk pairs; edges with whole-chunk cos ≥ `graph.near_dup_sim
|
|
|
108
110
|
Query side untouched. In subchunk mode `graph.k` means links per SUB-chunk (no per-chunk cap —
|
|
109
111
|
edge budget scales with concept count; graph gets ~1.4x denser at the 256-tok default). The
|
|
110
112
|
manifest guards `edge_source`/`subchunk_size_tokens` like the embedding model — an incremental
|
|
111
|
-
`index` after a flip fails loud, naming the old/new value; `index --
|
|
112
|
-
|
|
113
|
+
`index` after a flip fails loud, naming the old/new value; `index --edges-only` (`core/reedge.py`)
|
|
114
|
+
rebuilds just sub-chunks/edges/communities from the stored chunks — no re-chunk/re-embed, reusing
|
|
115
|
+
each whole chunk's stored vector — or `index --full` rebuilds everything, or pin the old value in
|
|
116
|
+
`ragx.toml`. Rationale + literature: `research/fine-grained-sub-chunk-edges-*.md`.
|
|
113
117
|
MEASURED 2026-07-12 (`.lab` experiments #10–#11): on the scoped notes eval corpus the dense
|
|
114
118
|
concept graph at hops2 TIES the tuned chunk-graph baseline (r@10 .923 / MRR .718 vs .717) at
|
|
115
119
|
~4.5x index embed cost (128-tok sub-chunks) — no win because the corpus's short single-topic
|
|
@@ -144,9 +148,9 @@ Each result chunk also carries grep-style `-B`/`-A`/`-C` context lines (`core/co
|
|
|
144
148
|
reads since no text is shown.
|
|
145
149
|
|
|
146
150
|
**Web** (`ragx-cli web`): builds the `ragx.graph.v1` payload once at startup (`graph_export.py`
|
|
147
|
-
—
|
|
148
|
-
it plus `/api/chunk/{id}` and `/api/query` (same pipeline as `query`,
|
|
149
|
-
cached, one query at a time behind a lock) over stdlib `http.server`.
|
|
151
|
+
— renders the index's stored edges, sub-chunk or chunk-mode cosine; fails loud only when no
|
|
152
|
+
index exists) and serves it plus `/api/chunk/{id}` and `/api/query` (same pipeline as `query`,
|
|
153
|
+
providers built lazily and cached, one query at a time behind a lock) over stdlib `http.server`.
|
|
150
154
|
|
|
151
155
|
## State & benchmarks (as of 2026-07-10)
|
|
152
156
|
|
|
@@ -201,7 +205,7 @@ mostly on Dutch/multilingual queries. Details: `research/bge-m3-dense-q8-vs-nomi
|
|
|
201
205
|
|
|
202
206
|
## Dev loop
|
|
203
207
|
|
|
204
|
-
`uv sync --group dev --extra rerank` · `uv run pytest -q` (
|
|
208
|
+
`uv sync --group dev --extra rerank` · `uv run pytest -q` (300 pass + 1 skipped, ~9 s) ·
|
|
205
209
|
`uv run ruff check src tests` · file soft cap ~150 lines · expected failures raise `RagxError`
|
|
206
210
|
(CLI maps to exit 2). Live smoke: LM Studio must be running with the configured embedding model.
|
|
207
211
|
Changes that impact usage (CLI flags, config keys/precedence, output schemas, install steps)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ragx-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.10.0
|
|
4
4
|
Summary: Similarity-graph RAG CLI: index a corpus into a chunk-level embedding graph and query it
|
|
5
5
|
Project-URL: Repository, https://github.com/alber70g/ragx-cli
|
|
6
6
|
Project-URL: Issues, https://github.com/alber70g/ragx-cli/issues
|
|
@@ -52,6 +52,7 @@ ragx-cli init # create ragx.toml next to your corpus (interacti
|
|
|
52
52
|
ragx-cli index # chunk -> embed -> HNSW + kNN similarity graph (incremental after the first run)
|
|
53
53
|
ragx-cli query "why did we switch build tools?" --json --files-only
|
|
54
54
|
ragx-cli index --full # full rebuild: re-chunk + re-embed everything
|
|
55
|
+
ragx-cli index --edges-only # rebuild sub-chunks/edges/communities only; keeps chunk embeddings
|
|
55
56
|
ragx-cli models # recommend + download an embedding/reranker combo via LM Studio
|
|
56
57
|
ragx-cli doctor # check the configured providers start and answer, and the index matches
|
|
57
58
|
ragx-cli --version # version, repo, and the effective models (corpus config + ~/.ragxrc overrides)
|
|
@@ -97,8 +98,10 @@ chunk mixing several concepts gets one sharp edge per concept instead of a dilut
|
|
|
97
98
|
between near-duplicate chunks (whole-chunk cosine ≥ `near_dup_sim`) are dropped — those are the
|
|
98
99
|
measured precision-killers. Costs ~2× more embedding calls at index time than `edge_source =
|
|
99
100
|
"chunk"`; retrieval units, traversal, and query flow are unchanged. Switching `edge_source` (or
|
|
100
|
-
the sub-chunk size) requires `index --
|
|
101
|
-
the
|
|
101
|
+
the sub-chunk size) requires either `index --edges-only` (cheap: rebuilds sub-chunks, edges and
|
|
102
|
+
communities from the stored chunks — chunk embeddings and the HNSW index are kept) or
|
|
103
|
+
`index --full`; incremental runs fail loud on the mismatch, naming the old/new value and both
|
|
104
|
+
fixes. Literature grounding:
|
|
102
105
|
`research/fine-grained-sub-chunk-edges-with-coarse-chunk-nodes-multi-granularity-graph-rag-literature-validation.md`.
|
|
103
106
|
In this mode `k` counts links per *sub-chunk* with no per-chunk cap, so the graph is denser and
|
|
104
107
|
shallower traversal suffices (the default `hops=2` already matches this). Quality ties
|
|
@@ -129,17 +132,24 @@ flowchart TD
|
|
|
129
132
|
re-embeds only new/modified files, drops deleted ones, and repairs only the edge lists those
|
|
130
133
|
chunks touch. Content hashes make `touch`-ed but unchanged files free, and include/exclude glob
|
|
131
134
|
changes converge to the same file set a rebuild would produce. `--full` rebuilds from scratch;
|
|
132
|
-
changing `embeddings.model
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
135
|
+
changing `embeddings.model` or `chunking.*` requires it — incremental runs fail loud on the
|
|
136
|
+
manifest mismatch, naming the changed key and its old/new value. Changing `graph.edge_source`/
|
|
137
|
+
`subchunk_size_tokens` alone is cheaper: `--edges-only` rebuilds just the sub-chunks, edges and
|
|
138
|
+
communities from the chunks already stored — it skips chunking and embedding entirely, reusing
|
|
139
|
+
each chunk's stored vector (via the HNSW index) for chunks that stay whole, and only calls the
|
|
140
|
+
embedder for the sub-chunk pieces a long chunk splits into. It fails loud the same way `--full`
|
|
141
|
+
does if `embeddings.model`/`chunking.*` don't match the stored manifest (those invalidate the
|
|
142
|
+
chunks it reuses), but `edge_source`/`subchunk_size_tokens` are exactly what it's meant to change.
|
|
143
|
+
`ragx-cli status` reports **drift** (files new/changed/deleted since the last index) so you can
|
|
144
|
+
see when a re-index is due without running one.
|
|
136
145
|
|
|
137
146
|
Upgrading from an older `ragx-cli` after a default change (e.g. the `graph.edge_source` flip to
|
|
138
147
|
`"subchunk"`): `ragx-cli init` always writes every `[graph]` key explicitly into `ragx.toml`
|
|
139
148
|
(`Config.save` bakes the full default set, not just overrides), so an init-created corpus already
|
|
140
149
|
pins its `edge_source`/`subchunk_size_tokens` and is unaffected by a later default change. Only a
|
|
141
150
|
hand-written `ragx.toml` missing those keys picks up the new default; its next incremental
|
|
142
|
-
`index` then fails loud — `index --
|
|
151
|
+
`index` then fails loud — `index --edges-only` rebuilds just the graph (cheapest), `index --full`
|
|
152
|
+
rebuilds everything, or pin the old value back in `ragx.toml`.
|
|
143
153
|
|
|
144
154
|
### Querying
|
|
145
155
|
|
|
@@ -477,11 +487,11 @@ anything. If a run dies with "connection refused", `doctor` names the stage and
|
|
|
477
487
|
ragx-cli web [--host 127.0.0.1] [--port 8765] [--open/--no-open]
|
|
478
488
|
```
|
|
479
489
|
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
cleanly.
|
|
490
|
+
Renders whatever edges the index has: sub-chunk similarity when it was built with
|
|
491
|
+
`graph.edge_source = "subchunk"` (the default), whole-chunk cosine when `"chunk"`. Fails loud
|
|
492
|
+
(exit 2) only when no index exists yet. To switch an existing index to sub-chunk edges without a
|
|
493
|
+
full re-embed, set the `[graph]` keys in `ragx.toml` and run `ragx-cli index --edges-only`.
|
|
494
|
+
`--open` (default) launches the browser; Ctrl-C shuts the server down cleanly.
|
|
485
495
|
|
|
486
496
|
Endpoints (JSON; errors are `{"error": "<message>"}`):
|
|
487
497
|
|
|
@@ -24,6 +24,7 @@ ragx-cli init # create ragx.toml next to your corpus (interacti
|
|
|
24
24
|
ragx-cli index # chunk -> embed -> HNSW + kNN similarity graph (incremental after the first run)
|
|
25
25
|
ragx-cli query "why did we switch build tools?" --json --files-only
|
|
26
26
|
ragx-cli index --full # full rebuild: re-chunk + re-embed everything
|
|
27
|
+
ragx-cli index --edges-only # rebuild sub-chunks/edges/communities only; keeps chunk embeddings
|
|
27
28
|
ragx-cli models # recommend + download an embedding/reranker combo via LM Studio
|
|
28
29
|
ragx-cli doctor # check the configured providers start and answer, and the index matches
|
|
29
30
|
ragx-cli --version # version, repo, and the effective models (corpus config + ~/.ragxrc overrides)
|
|
@@ -69,8 +70,10 @@ chunk mixing several concepts gets one sharp edge per concept instead of a dilut
|
|
|
69
70
|
between near-duplicate chunks (whole-chunk cosine ≥ `near_dup_sim`) are dropped — those are the
|
|
70
71
|
measured precision-killers. Costs ~2× more embedding calls at index time than `edge_source =
|
|
71
72
|
"chunk"`; retrieval units, traversal, and query flow are unchanged. Switching `edge_source` (or
|
|
72
|
-
the sub-chunk size) requires `index --
|
|
73
|
-
the
|
|
73
|
+
the sub-chunk size) requires either `index --edges-only` (cheap: rebuilds sub-chunks, edges and
|
|
74
|
+
communities from the stored chunks — chunk embeddings and the HNSW index are kept) or
|
|
75
|
+
`index --full`; incremental runs fail loud on the mismatch, naming the old/new value and both
|
|
76
|
+
fixes. Literature grounding:
|
|
74
77
|
`research/fine-grained-sub-chunk-edges-with-coarse-chunk-nodes-multi-granularity-graph-rag-literature-validation.md`.
|
|
75
78
|
In this mode `k` counts links per *sub-chunk* with no per-chunk cap, so the graph is denser and
|
|
76
79
|
shallower traversal suffices (the default `hops=2` already matches this). Quality ties
|
|
@@ -101,17 +104,24 @@ flowchart TD
|
|
|
101
104
|
re-embeds only new/modified files, drops deleted ones, and repairs only the edge lists those
|
|
102
105
|
chunks touch. Content hashes make `touch`-ed but unchanged files free, and include/exclude glob
|
|
103
106
|
changes converge to the same file set a rebuild would produce. `--full` rebuilds from scratch;
|
|
104
|
-
changing `embeddings.model
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
107
|
+
changing `embeddings.model` or `chunking.*` requires it — incremental runs fail loud on the
|
|
108
|
+
manifest mismatch, naming the changed key and its old/new value. Changing `graph.edge_source`/
|
|
109
|
+
`subchunk_size_tokens` alone is cheaper: `--edges-only` rebuilds just the sub-chunks, edges and
|
|
110
|
+
communities from the chunks already stored — it skips chunking and embedding entirely, reusing
|
|
111
|
+
each chunk's stored vector (via the HNSW index) for chunks that stay whole, and only calls the
|
|
112
|
+
embedder for the sub-chunk pieces a long chunk splits into. It fails loud the same way `--full`
|
|
113
|
+
does if `embeddings.model`/`chunking.*` don't match the stored manifest (those invalidate the
|
|
114
|
+
chunks it reuses), but `edge_source`/`subchunk_size_tokens` are exactly what it's meant to change.
|
|
115
|
+
`ragx-cli status` reports **drift** (files new/changed/deleted since the last index) so you can
|
|
116
|
+
see when a re-index is due without running one.
|
|
108
117
|
|
|
109
118
|
Upgrading from an older `ragx-cli` after a default change (e.g. the `graph.edge_source` flip to
|
|
110
119
|
`"subchunk"`): `ragx-cli init` always writes every `[graph]` key explicitly into `ragx.toml`
|
|
111
120
|
(`Config.save` bakes the full default set, not just overrides), so an init-created corpus already
|
|
112
121
|
pins its `edge_source`/`subchunk_size_tokens` and is unaffected by a later default change. Only a
|
|
113
122
|
hand-written `ragx.toml` missing those keys picks up the new default; its next incremental
|
|
114
|
-
`index` then fails loud — `index --
|
|
123
|
+
`index` then fails loud — `index --edges-only` rebuilds just the graph (cheapest), `index --full`
|
|
124
|
+
rebuilds everything, or pin the old value back in `ragx.toml`.
|
|
115
125
|
|
|
116
126
|
### Querying
|
|
117
127
|
|
|
@@ -449,11 +459,11 @@ anything. If a run dies with "connection refused", `doctor` names the stage and
|
|
|
449
459
|
ragx-cli web [--host 127.0.0.1] [--port 8765] [--open/--no-open]
|
|
450
460
|
```
|
|
451
461
|
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
cleanly.
|
|
462
|
+
Renders whatever edges the index has: sub-chunk similarity when it was built with
|
|
463
|
+
`graph.edge_source = "subchunk"` (the default), whole-chunk cosine when `"chunk"`. Fails loud
|
|
464
|
+
(exit 2) only when no index exists yet. To switch an existing index to sub-chunk edges without a
|
|
465
|
+
full re-embed, set the `[graph]` keys in `ragx.toml` and run `ragx-cli index --edges-only`.
|
|
466
|
+
`--open` (default) launches the browser; Ctrl-C shuts the server down cleanly.
|
|
457
467
|
|
|
458
468
|
Endpoints (JSON; errors are `{"error": "<message>"}`):
|
|
459
469
|
|
|
@@ -35,6 +35,7 @@ ragx-cli query "..." --json --rerank # higher precision: cross-encode
|
|
|
35
35
|
echo "..." | ragx-cli query - --json # long/multiline query from stdin
|
|
36
36
|
ragx-cli index # incremental reindex (hash-based, cheap) — the default
|
|
37
37
|
ragx-cli index --full # full rebuild: re-chunks + re-embeds everything
|
|
38
|
+
ragx-cli index --edges-only # rebuild sub-chunks/edges/communities only; keeps embeddings
|
|
38
39
|
```
|
|
39
40
|
|
|
40
41
|
Other query flags: `--expand` (one LLM call: multi-query + HyDE), `--no-graph` (pure vector),
|
|
@@ -60,8 +61,10 @@ default 5). `--no-expand`/`--no-rerank` are deprecated no-ops (both stages are o
|
|
|
60
61
|
(incremental, safe, converges even after include/exclude glob changes).
|
|
61
62
|
- Changing `embeddings.model`, `chunking.*`, or `graph.edge_source`/`subchunk_size_tokens`
|
|
62
63
|
invalidates the index: commands fail loud on the manifest mismatch, naming the changed
|
|
63
|
-
key and old/new value.
|
|
64
|
-
|
|
64
|
+
key and old/new value. For an `edge_source`/`subchunk_size_tokens` change, `ragx-cli index
|
|
65
|
+
--edges-only` is the cheap fix — rebuilds sub-chunks/edges/communities, keeps chunk
|
|
66
|
+
embeddings. Otherwise `ragx-cli index --full` — re-embeds everything, so confirm before
|
|
67
|
+
running on a large corpus — or pin the old value back in `ragx.toml`.
|
|
65
68
|
- `ragx-cli models` picks + downloads an embedding/reranker combo. All files download via
|
|
66
69
|
LM Studio; serving is per engine — `--embed-engine llama-server|lm-studio` and
|
|
67
70
|
`--rerank-engine llama-server|sentence-transformers`. With both on `llama-server`
|
|
@@ -12,6 +12,7 @@ from ragx.core.config import Config, require_root
|
|
|
12
12
|
from ragx.core.errors import RagxError
|
|
13
13
|
from ragx.core.indexer import run_index
|
|
14
14
|
from ragx.core.query import QueryOptions, run_query, to_files_json, to_query_json
|
|
15
|
+
from ragx.core.reedge import rebuild_edges
|
|
15
16
|
from ragx.providers import lmstudio_process
|
|
16
17
|
from ragx.providers.registry import make_embedder, make_generator, make_reranker
|
|
17
18
|
|
|
@@ -24,6 +25,12 @@ def register(app: typer.Typer) -> None:
|
|
|
24
25
|
def index(
|
|
25
26
|
path: Path | None = typer.Argument(None),
|
|
26
27
|
full: bool = typer.Option(False, "--full", help="full rebuild: re-chunk and re-embed everything"),
|
|
28
|
+
edges_only: bool = typer.Option(
|
|
29
|
+
False,
|
|
30
|
+
"--edges-only",
|
|
31
|
+
help="rebuild sub-chunks, graph edges and communities from the stored chunks per "
|
|
32
|
+
"the current [graph] config; chunk embeddings and the vector index are kept",
|
|
33
|
+
),
|
|
27
34
|
changed: bool = typer.Option(
|
|
28
35
|
False, "--changed", hidden=True, help="deprecated: incremental is now the default"
|
|
29
36
|
),
|
|
@@ -32,13 +39,20 @@ def index(
|
|
|
32
39
|
"""Chunk, embed, and index the corpus (incremental by default; --full rebuilds)."""
|
|
33
40
|
if changed and full:
|
|
34
41
|
fail("--changed and --full are mutually exclusive")
|
|
42
|
+
if edges_only and full:
|
|
43
|
+
fail("--edges-only and --full are mutually exclusive")
|
|
44
|
+
if edges_only and changed:
|
|
45
|
+
fail("--edges-only and --changed are mutually exclusive")
|
|
35
46
|
if changed:
|
|
36
47
|
print("warning: --changed is deprecated; incremental is now the default", file=sys.stderr)
|
|
37
48
|
try:
|
|
38
49
|
root = require_root(path)
|
|
39
50
|
cfg = Config.load(root, confirm=migrate_confirm())
|
|
40
51
|
lmstudio_process.ensure_for_section(cfg, "embeddings")
|
|
41
|
-
|
|
52
|
+
if edges_only:
|
|
53
|
+
stats = rebuild_edges(root, cfg, make_embedder(cfg))
|
|
54
|
+
else:
|
|
55
|
+
stats = run_index(root, cfg, make_embedder(cfg), full=full)
|
|
42
56
|
except RagxError as exc:
|
|
43
57
|
fail(str(exc))
|
|
44
58
|
doc = {"schema": "ragx.index.v1", **stats.__dict__}
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
"""Builds the ragx.graph.v1 payload (nodes/edges/communities) for `ragx-cli web`.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
Renders whatever edges the index has: sub-chunk similarity when it was built with
|
|
4
|
+
graph.edge_source="subchunk", whole-chunk cosine when edge_source="chunk". An index with
|
|
5
|
+
zero edges (nodes only) is valid — the viewer just shows an unconnected cloud."""
|
|
6
6
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
from pathlib import Path
|
|
10
10
|
|
|
11
|
-
from ragx.core.errors import RagxError
|
|
12
11
|
from ragx.core.store import Store
|
|
13
12
|
|
|
14
13
|
LABEL_MAX_CHARS = 80
|
|
@@ -16,12 +15,7 @@ LABEL_MAX_CHARS = 80
|
|
|
16
15
|
|
|
17
16
|
def build_graph(root: Path, store: Store) -> dict:
|
|
18
17
|
edge_source = store.get_meta("edge_source")
|
|
19
|
-
|
|
20
|
-
raise RagxError(
|
|
21
|
-
"ragx-cli web shows sub-chunk edges, but this index was built with "
|
|
22
|
-
f'edge_source="{edge_source}". Set graph.edge_source = "subchunk" in '
|
|
23
|
-
"ragx.toml and run ragx-cli index."
|
|
24
|
-
)
|
|
18
|
+
sub_size = store.get_meta("subchunk_size_tokens")
|
|
25
19
|
|
|
26
20
|
chunks = store.get_chunks(store.all_chunk_ids())
|
|
27
21
|
edges = store.all_edges()
|
|
@@ -51,7 +45,7 @@ def build_graph(root: Path, store: Store) -> dict:
|
|
|
51
45
|
"meta": {
|
|
52
46
|
"root": root.name,
|
|
53
47
|
"edge_source": edge_source,
|
|
54
|
-
"subchunk_size_tokens": int(
|
|
48
|
+
"subchunk_size_tokens": int(sub_size) if edge_source == "subchunk" and sub_size else None,
|
|
55
49
|
"embedding_model": store.get_meta("embedding_model"),
|
|
56
50
|
"nodes": len(nodes),
|
|
57
51
|
"edges": len(edge_list),
|
|
@@ -16,7 +16,7 @@ from ragx.core.config import Config, db_path, vectors_path
|
|
|
16
16
|
from ragx.core.discovery import discover_files, hash_file
|
|
17
17
|
from ragx.core.errors import ManifestMismatchError, RagxError
|
|
18
18
|
from ragx.core.graph import affected_ids, knn_edges, subchunk_knn_edges
|
|
19
|
-
from ragx.core.models import
|
|
19
|
+
from ragx.core.models import FileRecord
|
|
20
20
|
from ragx.core.store import Store
|
|
21
21
|
from ragx.core.vectors import VectorIndex
|
|
22
22
|
from ragx.providers.base import Embedder
|
|
@@ -94,7 +94,7 @@ def run_index(root: Path, cfg: Config, embedder: Embedder, *, full: bool = False
|
|
|
94
94
|
vectors = embedder.embed_documents([d.text for d in drafts])
|
|
95
95
|
index.add(ids, vectors)
|
|
96
96
|
if edge_source == "subchunk":
|
|
97
|
-
_embed_subchunks(store, embedder, ids, drafts, vectors, sub_size)
|
|
97
|
+
_embed_subchunks(store, embedder, ids, [d.text for d in drafts], vectors, sub_size)
|
|
98
98
|
new_ids.extend(ids)
|
|
99
99
|
stats.files_indexed += 1
|
|
100
100
|
stats.chunks_added += len(ids)
|
|
@@ -150,9 +150,15 @@ def _check_manifest(
|
|
|
150
150
|
for key, current in checks:
|
|
151
151
|
prev = store.get_meta(key)
|
|
152
152
|
if prev and prev != str(current) and not allow_rewrite:
|
|
153
|
+
rebuild_hint = "run `ragx-cli index --full` to rebuild"
|
|
154
|
+
if key in ("edge_source", "subchunk_size_tokens"):
|
|
155
|
+
rebuild_hint = (
|
|
156
|
+
"run `ragx-cli index --edges-only` to rebuild just the graph (cheaper: "
|
|
157
|
+
"keeps chunk embeddings), or `ragx-cli index --full` to rebuild everything"
|
|
158
|
+
)
|
|
153
159
|
raise ManifestMismatchError(
|
|
154
160
|
f"index was built with {key}={prev!r} but config says {current!r}; "
|
|
155
|
-
"
|
|
161
|
+
f"{rebuild_hint}, or pin the old value "
|
|
156
162
|
f"({key} = {prev!r}) in ragx.toml to keep the existing index"
|
|
157
163
|
)
|
|
158
164
|
|
|
@@ -192,18 +198,21 @@ def _embed_subchunks(
|
|
|
192
198
|
store: Store,
|
|
193
199
|
embedder: Embedder,
|
|
194
200
|
ids: Sequence[int],
|
|
195
|
-
|
|
201
|
+
texts: Sequence[str],
|
|
196
202
|
chunk_vectors: Sequence[Sequence[float]],
|
|
197
203
|
sub_size: int,
|
|
198
|
-
) ->
|
|
199
|
-
"""Split each chunk into sub-chunks and persist their unit vectors.
|
|
200
|
-
A chunk that stays whole reuses its own embedding — no extra provider call.
|
|
204
|
+
) -> tuple[int, int]:
|
|
205
|
+
"""Split each chunk's text into sub-chunks and persist their unit vectors.
|
|
206
|
+
A chunk that stays whole reuses its own embedding — no extra provider call.
|
|
207
|
+
Returns (sub-chunks embedded, sub-chunks reused from the chunk embedding)."""
|
|
201
208
|
to_embed: list[str] = []
|
|
202
209
|
spans: list[tuple[int, int, int]] = [] # (chunk_id, start, end) into to_embed
|
|
203
|
-
|
|
204
|
-
|
|
210
|
+
reused = 0
|
|
211
|
+
for cid, text, cvec in zip(ids, texts, chunk_vectors):
|
|
212
|
+
subs = subchunk_texts(text, sub_size)
|
|
205
213
|
if len(subs) == 1:
|
|
206
214
|
store.insert_subchunk_vectors(cid, [_unit(cvec)])
|
|
215
|
+
reused += 1
|
|
207
216
|
else:
|
|
208
217
|
spans.append((cid, len(to_embed), len(to_embed) + len(subs)))
|
|
209
218
|
to_embed.extend(subs)
|
|
@@ -211,6 +220,7 @@ def _embed_subchunks(
|
|
|
211
220
|
vectors = embedder.embed_documents(to_embed)
|
|
212
221
|
for cid, start, end in spans:
|
|
213
222
|
store.insert_subchunk_vectors(cid, [_unit(v) for v in vectors[start:end]])
|
|
223
|
+
return len(to_embed), reused
|
|
214
224
|
|
|
215
225
|
|
|
216
226
|
def _subchunk_edge_fn(
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Rebuild sub-chunks, graph edges and communities from stored chunks, without re-chunking
|
|
2
|
+
or re-embedding files. Backs `ragx-cli index --edges-only`, for switching `graph.edge_source`
|
|
3
|
+
(or `subchunk_size_tokens`) on an existing index without paying for a full re-embed."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from ragx.core.communities import leiden_communities
|
|
11
|
+
from ragx.core.config import Config, db_path
|
|
12
|
+
from ragx.core.errors import ManifestMismatchError, RagxError
|
|
13
|
+
from ragx.core.graph import knn_edges
|
|
14
|
+
from ragx.core.indexer import IndexStats, _embed_subchunks, _open_vectors, _subchunk_edge_fn
|
|
15
|
+
from ragx.core.store import Store
|
|
16
|
+
from ragx.providers.base import Embedder
|
|
17
|
+
|
|
18
|
+
log = logging.getLogger("ragx.index")
|
|
19
|
+
|
|
20
|
+
_BATCH = 300
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def rebuild_edges(root: Path, cfg: Config, embedder: Embedder) -> IndexStats:
|
|
24
|
+
"""Rebuild sub-chunks (if `graph.edge_source == "subchunk"`), graph edges and communities
|
|
25
|
+
from the chunks already stored at `root`, per the current [graph] config. Chunk embeddings
|
|
26
|
+
and the vector index are untouched: a chunk that stays whole reuses its own stored vector —
|
|
27
|
+
no provider call re-embeds it."""
|
|
28
|
+
edge_source = cfg.get("graph.edge_source")
|
|
29
|
+
if edge_source not in ("chunk", "subchunk"):
|
|
30
|
+
raise RagxError(f"graph.edge_source must be 'chunk' or 'subchunk', got {edge_source!r}")
|
|
31
|
+
sub_size = cfg.get("graph.subchunk_size_tokens")
|
|
32
|
+
size_tokens = cfg.get("chunking.size_tokens")
|
|
33
|
+
overlap = cfg.get("chunking.overlap")
|
|
34
|
+
|
|
35
|
+
if not db_path(root).exists():
|
|
36
|
+
raise RagxError("no index found — run `ragx-cli index` first")
|
|
37
|
+
with Store(db_path(root)) as store:
|
|
38
|
+
if store.chunk_count() == 0:
|
|
39
|
+
raise RagxError("no index found — run `ragx-cli index` first")
|
|
40
|
+
_check_manifest(store, embedder, size_tokens, overlap)
|
|
41
|
+
|
|
42
|
+
dim = embedder.dimension()
|
|
43
|
+
index = _open_vectors(root, dim)
|
|
44
|
+
store.clear_subchunks()
|
|
45
|
+
store.clear_edges()
|
|
46
|
+
|
|
47
|
+
all_ids = store.all_chunk_ids()
|
|
48
|
+
k, min_sim = cfg.get("graph.k"), cfg.get("graph.min_edge_sim")
|
|
49
|
+
embedded = reused = 0
|
|
50
|
+
if edge_source == "subchunk":
|
|
51
|
+
for i in range(0, len(all_ids), _BATCH):
|
|
52
|
+
batch = all_ids[i : i + _BATCH]
|
|
53
|
+
texts = [c.text for c in store.get_chunks(batch)]
|
|
54
|
+
vectors = index.get_vectors(batch)
|
|
55
|
+
n_embed, n_reuse = _embed_subchunks(store, embedder, batch, texts, vectors, sub_size)
|
|
56
|
+
embedded += n_embed
|
|
57
|
+
reused += n_reuse
|
|
58
|
+
edge_fn = _subchunk_edge_fn(store, index, dim, k, min_sim, cfg.get("graph.near_dup_sim"))
|
|
59
|
+
else:
|
|
60
|
+
def edge_fn(ids_: list[int]) -> dict[int, list[tuple[int, float]]]:
|
|
61
|
+
return knn_edges(index, ids_, k, min_sim)
|
|
62
|
+
|
|
63
|
+
edges = edge_fn(all_ids)
|
|
64
|
+
for src, nbrs in edges.items():
|
|
65
|
+
store.replace_edges(src, nbrs)
|
|
66
|
+
|
|
67
|
+
assignments = leiden_communities(
|
|
68
|
+
store.all_edges(),
|
|
69
|
+
resolution=cfg.get("communities.resolution"),
|
|
70
|
+
seed=cfg.get("communities.seed"),
|
|
71
|
+
)
|
|
72
|
+
store.replace_communities(assignments)
|
|
73
|
+
|
|
74
|
+
store.set_meta("edge_source", edge_source)
|
|
75
|
+
store.set_meta("subchunk_size_tokens", str(sub_size))
|
|
76
|
+
|
|
77
|
+
stats = IndexStats(
|
|
78
|
+
edges_total=store.edge_count(),
|
|
79
|
+
communities_total=len(set(assignments.values())),
|
|
80
|
+
)
|
|
81
|
+
if edge_source == "subchunk":
|
|
82
|
+
log.info("sub-chunks: %d embedded, %d reused", embedded, reused)
|
|
83
|
+
log.info(
|
|
84
|
+
"edges-only rebuild: %d chunks, %d edges, %d communities",
|
|
85
|
+
len(all_ids), stats.edges_total, stats.communities_total,
|
|
86
|
+
)
|
|
87
|
+
return stats
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _check_manifest(store: Store, embedder: Embedder, size_tokens: int, overlap: float) -> None:
|
|
91
|
+
"""embedding_model/chunk_size_tokens/chunk_overlap must match: edges are rebuilt from
|
|
92
|
+
stored chunk texts and reused chunk embeddings, which are only valid for the embedder and
|
|
93
|
+
chunking the index was built with. edge_source/subchunk_size_tokens are NOT checked here —
|
|
94
|
+
changing them is the point of --edges-only."""
|
|
95
|
+
checks = [
|
|
96
|
+
("embedding_model", embedder.model),
|
|
97
|
+
("chunk_size_tokens", str(size_tokens)),
|
|
98
|
+
("chunk_overlap", str(overlap)),
|
|
99
|
+
]
|
|
100
|
+
for key, current in checks:
|
|
101
|
+
prev = store.get_meta(key)
|
|
102
|
+
if prev and prev != str(current):
|
|
103
|
+
raise ManifestMismatchError(
|
|
104
|
+
f"index was built with {key}={prev!r} but config says {current!r}; "
|
|
105
|
+
"run `ragx-cli index --full` to rebuild"
|
|
106
|
+
)
|
|
@@ -166,6 +166,10 @@ class Store:
|
|
|
166
166
|
"SELECT chunk_id, vector FROM subchunks ORDER BY chunk_id, id"
|
|
167
167
|
).fetchall()
|
|
168
168
|
|
|
169
|
+
def clear_subchunks(self) -> None:
|
|
170
|
+
self.conn.execute("DELETE FROM subchunks")
|
|
171
|
+
self.conn.commit()
|
|
172
|
+
|
|
169
173
|
# -- edges (undirected; stored normalized src < dst) -----------------
|
|
170
174
|
|
|
171
175
|
def replace_edges(self, src: int, neighbors: Sequence[tuple[int, float]]) -> None:
|
|
@@ -199,6 +203,10 @@ class Store:
|
|
|
199
203
|
def edge_count(self) -> int:
|
|
200
204
|
return self.conn.execute("SELECT COUNT(*) FROM edges").fetchone()[0]
|
|
201
205
|
|
|
206
|
+
def clear_edges(self) -> None:
|
|
207
|
+
self.conn.execute("DELETE FROM edges")
|
|
208
|
+
self.conn.commit()
|
|
209
|
+
|
|
202
210
|
def all_edges(self) -> list[tuple[int, int, float]]:
|
|
203
211
|
return self.conn.execute("SELECT src, dst, weight FROM edges ORDER BY src, dst").fetchall()
|
|
204
212
|
|
|
@@ -63,6 +63,7 @@ export function savePositions(graph, key) {
|
|
|
63
63
|
function settingsFor(graph) {
|
|
64
64
|
const settings = FA2.inferSettings(graph);
|
|
65
65
|
settings.barnesHutOptimize = graph.order > 2000;
|
|
66
|
+
settings.edgeWeightInfluence = 1; // edge `weight` (similarity) pulls stronger links closer
|
|
66
67
|
return settings;
|
|
67
68
|
}
|
|
68
69
|
|
|
@@ -23,12 +23,15 @@ function showPanel(nodes) {
|
|
|
23
23
|
}
|
|
24
24
|
|
|
25
25
|
function renderMeta(meta) {
|
|
26
|
+
const edgeSource = meta.edge_source === "subchunk"
|
|
27
|
+
? `subchunk (${meta.subchunk_size_tokens} tok)`
|
|
28
|
+
: "chunk (whole-chunk cosine)";
|
|
26
29
|
const items = [
|
|
27
30
|
["corpus", meta.root],
|
|
28
31
|
["chunks", fmtInt(meta.nodes)],
|
|
29
32
|
["edges", fmtInt(meta.edges)],
|
|
30
33
|
["communities", fmtInt(meta.communities)],
|
|
31
|
-
["edge source",
|
|
34
|
+
["edge source", edgeSource],
|
|
32
35
|
["embedding", meta.embedding_model],
|
|
33
36
|
];
|
|
34
37
|
$("#meta").replaceChildren(...items.map(([k, v]) =>
|
|
@@ -251,6 +251,20 @@ def test_index_changed_and_full_mutually_exclusive(tmp_path, monkeypatch):
|
|
|
251
251
|
assert result.exit_code == 2
|
|
252
252
|
|
|
253
253
|
|
|
254
|
+
def test_index_edges_only_and_full_mutually_exclusive(tmp_path, monkeypatch):
|
|
255
|
+
runner.invoke(app, ["init", str(tmp_path)])
|
|
256
|
+
monkeypatch.chdir(tmp_path)
|
|
257
|
+
result = runner.invoke(app, ["index", "--edges-only", "--full"])
|
|
258
|
+
assert result.exit_code == 2
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def test_index_edges_only_and_changed_mutually_exclusive(tmp_path, monkeypatch):
|
|
262
|
+
runner.invoke(app, ["init", str(tmp_path)])
|
|
263
|
+
monkeypatch.chdir(tmp_path)
|
|
264
|
+
result = runner.invoke(app, ["index", "--edges-only", "--changed"])
|
|
265
|
+
assert result.exit_code == 2
|
|
266
|
+
|
|
267
|
+
|
|
254
268
|
def test_status_without_init(tmp_path, monkeypatch):
|
|
255
269
|
monkeypatch.chdir(tmp_path)
|
|
256
270
|
result = runner.invoke(app, ["status"])
|