cairn-retrieval 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cairn_retrieval-0.1.0/.gitignore +36 -0
- cairn_retrieval-0.1.0/LICENSE +21 -0
- cairn_retrieval-0.1.0/PKG-INFO +68 -0
- cairn_retrieval-0.1.0/README.md +45 -0
- cairn_retrieval-0.1.0/intents/adaptive_gate_intent.yaml +142 -0
- cairn_retrieval-0.1.0/intents/assembler_intent.yaml +135 -0
- cairn_retrieval-0.1.0/intents/connection_suggestion_intent.yaml +162 -0
- cairn_retrieval-0.1.0/intents/fusion_intent.yaml +126 -0
- cairn_retrieval-0.1.0/intents/lexical_signal_intent.yaml +135 -0
- cairn_retrieval-0.1.0/intents/link_ranking_intent.yaml +142 -0
- cairn_retrieval-0.1.0/intents/retrieve_e2e_intent.yaml +131 -0
- cairn_retrieval-0.1.0/intents/semantic_signal_intent.yaml +134 -0
- cairn_retrieval-0.1.0/pyproject.toml +37 -0
- cairn_retrieval-0.1.0/src/cairn_retrieval/__init__.py +77 -0
- cairn_retrieval-0.1.0/src/cairn_retrieval/assembler.py +113 -0
- cairn_retrieval-0.1.0/src/cairn_retrieval/connections.py +140 -0
- cairn_retrieval-0.1.0/src/cairn_retrieval/fusion.py +61 -0
- cairn_retrieval-0.1.0/src/cairn_retrieval/gate.py +140 -0
- cairn_retrieval-0.1.0/src/cairn_retrieval/lexical.py +129 -0
- cairn_retrieval-0.1.0/src/cairn_retrieval/link_ranking.py +134 -0
- cairn_retrieval-0.1.0/src/cairn_retrieval/py.typed +0 -0
- cairn_retrieval-0.1.0/src/cairn_retrieval/retrieve.py +131 -0
- cairn_retrieval-0.1.0/src/cairn_retrieval/semantic.py +86 -0
- cairn_retrieval-0.1.0/tests/unit/test_assembler.py +113 -0
- cairn_retrieval-0.1.0/tests/unit/test_connections.py +161 -0
- cairn_retrieval-0.1.0/tests/unit/test_fusion.py +107 -0
- cairn_retrieval-0.1.0/tests/unit/test_gate.py +103 -0
- cairn_retrieval-0.1.0/tests/unit/test_lexical.py +109 -0
- cairn_retrieval-0.1.0/tests/unit/test_link_ranking.py +139 -0
- cairn_retrieval-0.1.0/tests/unit/test_retrieve.py +179 -0
- cairn_retrieval-0.1.0/tests/unit/test_semantic.py +117 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
*.so
|
|
9
|
+
|
|
10
|
+
# venv / uv
|
|
11
|
+
.venv/
|
|
12
|
+
.uv/
|
|
13
|
+
uv.lock
|
|
14
|
+
|
|
15
|
+
# tooling caches
|
|
16
|
+
.pytest_cache/
|
|
17
|
+
.mypy_cache/
|
|
18
|
+
.ruff_cache/
|
|
19
|
+
.coverage
|
|
20
|
+
htmlcov/
|
|
21
|
+
|
|
22
|
+
# editors / os
|
|
23
|
+
.vscode/
|
|
24
|
+
.idea/
|
|
25
|
+
.DS_Store
|
|
26
|
+
|
|
27
|
+
# local scratch / data (never commit real corpora)
|
|
28
|
+
*.db
|
|
29
|
+
*.sqlite
|
|
30
|
+
*.parquet
|
|
31
|
+
scratch/
|
|
32
|
+
tmp/
|
|
33
|
+
|
|
34
|
+
# user corpora (never ship user content in the library repo)
|
|
35
|
+
corpus/
|
|
36
|
+
examples/graph.svg # generated by graph_visualize.py — contains user corpus content
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Leo Celis
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cairn-retrieval
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Agent-retrieval layer on top of the Cairn entity engine — adaptive gate, signal orchestration (lexical/semantic/graph), and minimal context assembly. Zero generative-LLM on the hot path.
|
|
5
|
+
Project-URL: Homepage, https://github.com/leocelis/cairn
|
|
6
|
+
Project-URL: Source, https://github.com/leocelis/cairn
|
|
7
|
+
Project-URL: Issues, https://github.com/leocelis/cairn/issues
|
|
8
|
+
Author-email: Leo Celis <hello@leocelis.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: ai-agents,context-assembly,deterministic,rag,reranking,retrieval
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Requires-Dist: cairn-engine~=0.1.0
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# cairn-retrieval
|
|
25
|
+
|
|
26
|
+
**The agent-retrieval layer on top of the [`cairn-engine`](https://github.com/leocelis/cairn/tree/main/packages/cairn-engine) entity engine.**
|
|
27
|
+
|
|
28
|
+
The three retrieval primitives above the entity map — all deterministic, all
|
|
29
|
+
generative-LLM-free on the default path:
|
|
30
|
+
|
|
31
|
+
- **A · Adaptive gate** (`gate`) — two-stage, LLM-free decision *before* any
|
|
32
|
+
signal runs: deterministic bypass (temporal markers, multi-entity relational,
|
|
33
|
+
private entities) then a CA-RAG utility cost function. `strategy=none` (skip
|
|
34
|
+
retrieval) is a first-class, reachable outcome — selective retrieval beats
|
|
35
|
+
always-on empirically.
|
|
36
|
+
- **C · Signal orchestration** — `LexicalIndex` (pure-Python BM25) + `scan`
|
|
37
|
+
(exact/regex), opt-in `SemanticIndex` (caller-supplied embedder; no model SDK
|
|
38
|
+
imported), graph traversal via `cairn-engine`; ranked lists merged with `fuse`
|
|
39
|
+
(Reciprocal Rank Fusion, k=1, rank-only — no score normalization).
|
|
40
|
+
- **E · Context assembler** (`assemble`) — budget → MMR select → cosine dedup →
|
|
41
|
+
fold ordering (strongest evidence at the edges) → provenance-tagged emit with
|
|
42
|
+
a trace manifest. Never re-embeds; zero LLM.
|
|
43
|
+
|
|
44
|
+
`RetrievalEngine.retrieve(query, *, budget)` ties them end-to-end:
|
|
45
|
+
resolve → gate → orchestrate → fuse → assemble. A query the gate routes to
|
|
46
|
+
`none` returns no context at all — the skip is the point.
|
|
47
|
+
|
|
48
|
+
Also here: `rank_links` — TF-IDF-cosine document↔document link recommendation
|
|
49
|
+
over shared canonical concepts — and `suggest_connections` — cold-start link
|
|
50
|
+
prediction: rank the existing entities a NEW candidate node should connect to
|
|
51
|
+
(content-first, seed-optional structural signal, full provenance).
|
|
52
|
+
|
|
53
|
+
> Status: **feature-complete, 0.x.** Stays 0.x until the benchmark phase
|
|
54
|
+
> (OP-31 eval vs. agentic-grep / vector-RAG / Graphiti) proves the quality
|
|
55
|
+
> claims. Depends only on `cairn-engine` (which is itself zero-dependency). See
|
|
56
|
+
> the [roadmap](https://github.com/leocelis/cairn/blob/main/ROADMAP.md) and
|
|
57
|
+
> [`intents/`](https://github.com/leocelis/cairn/tree/main/packages/cairn-retrieval/intents)
|
|
58
|
+
> for the constraint-tested design of each module.
|
|
59
|
+
|
|
60
|
+
## Install
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install cairn-retrieval # pulls cairn-engine automatically
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## License
|
|
67
|
+
|
|
68
|
+
MIT — Copyright (c) 2026 Leo Celis
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# cairn-retrieval
|
|
2
|
+
|
|
3
|
+
**The agent-retrieval layer on top of the [`cairn-engine`](https://github.com/leocelis/cairn/tree/main/packages/cairn-engine) entity engine.**
|
|
4
|
+
|
|
5
|
+
The three retrieval primitives above the entity map — all deterministic, all
|
|
6
|
+
generative-LLM-free on the default path:
|
|
7
|
+
|
|
8
|
+
- **A · Adaptive gate** (`gate`) — two-stage, LLM-free decision *before* any
|
|
9
|
+
signal runs: deterministic bypass (temporal markers, multi-entity relational,
|
|
10
|
+
private entities) then a CA-RAG utility cost function. `strategy=none` (skip
|
|
11
|
+
retrieval) is a first-class, reachable outcome — selective retrieval beats
|
|
12
|
+
always-on empirically.
|
|
13
|
+
- **C · Signal orchestration** — `LexicalIndex` (pure-Python BM25) + `scan`
|
|
14
|
+
(exact/regex), opt-in `SemanticIndex` (caller-supplied embedder; no model SDK
|
|
15
|
+
imported), graph traversal via `cairn-engine`; ranked lists merged with `fuse`
|
|
16
|
+
(Reciprocal Rank Fusion, k=1, rank-only — no score normalization).
|
|
17
|
+
- **E · Context assembler** (`assemble`) — budget → MMR select → cosine dedup →
|
|
18
|
+
fold ordering (strongest evidence at the edges) → provenance-tagged emit with
|
|
19
|
+
a trace manifest. Never re-embeds; zero LLM.
|
|
20
|
+
|
|
21
|
+
`RetrievalEngine.retrieve(query, *, budget)` ties them end-to-end:
|
|
22
|
+
resolve → gate → orchestrate → fuse → assemble. A query the gate routes to
|
|
23
|
+
`none` returns no context at all — the skip is the point.
|
|
24
|
+
|
|
25
|
+
Also here: `rank_links` — TF-IDF-cosine document↔document link recommendation
|
|
26
|
+
over shared canonical concepts — and `suggest_connections` — cold-start link
|
|
27
|
+
prediction: rank the existing entities a NEW candidate node should connect to
|
|
28
|
+
(content-first, seed-optional structural signal, full provenance).
|
|
29
|
+
|
|
30
|
+
> Status: **feature-complete, 0.x.** Stays 0.x until the benchmark phase
|
|
31
|
+
> (OP-31 eval vs. agentic-grep / vector-RAG / Graphiti) proves the quality
|
|
32
|
+
> claims. Depends only on `cairn-engine` (which is itself zero-dependency). See
|
|
33
|
+
> the [roadmap](https://github.com/leocelis/cairn/blob/main/ROADMAP.md) and
|
|
34
|
+
> [`intents/`](https://github.com/leocelis/cairn/tree/main/packages/cairn-retrieval/intents)
|
|
35
|
+
> for the constraint-tested design of each module.
|
|
36
|
+
|
|
37
|
+
## Install
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip install cairn-retrieval # pulls cairn-engine automatically
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## License
|
|
44
|
+
|
|
45
|
+
MIT — Copyright (c) 2026 Leo Celis
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# ==============================================================================
|
|
2
|
+
# cairn-retrieval — Module Intent: adaptive gate (M3.1)
|
|
3
|
+
# ==============================================================================
|
|
4
|
+
# License: MIT · Copyright (c) 2026 Leo Celis · https://ivdframework.dev
|
|
5
|
+
# Parent: ../../../cairn_system_intent.yaml (v0.2.1)
|
|
6
|
+
# Patterns: patterns_retrieval_knowledge.yaml — OP-33 (gate cost model, PRIMARY),
|
|
7
|
+
# OP-1 (7-test when-to-retrieve), OP-25 (RAG paradigm router)
|
|
8
|
+
# ==============================================================================
|
|
9
|
+
|
|
10
|
+
scope:
|
|
11
|
+
level: "module"
|
|
12
|
+
type: "feature"
|
|
13
|
+
granularity: "fine"
|
|
14
|
+
|
|
15
|
+
metadata:
|
|
16
|
+
feature_id: "cairn_retrieval_adaptive_gate"
|
|
17
|
+
category: "agent_infrastructure"
|
|
18
|
+
tags: ["adaptive-gate", "when-to-retrieve", "routing", "llm-free", "deterministic"]
|
|
19
|
+
status: "in_progress"
|
|
20
|
+
introduced_version: "0.0.1"
|
|
21
|
+
|
|
22
|
+
intent:
|
|
23
|
+
summary: >
|
|
24
|
+
A two-stage, LLM-free gate that runs before retrieval and returns a
|
|
25
|
+
RoutingDecision (strategy ∈ none | grep | semantic | graph | composite).
|
|
26
|
+
Stage 1 is a deterministic bypass (temporal/version markers, multi-entity
|
|
27
|
+
relational, private entity). Stage 2 is the CA-RAG utility cost function
|
|
28
|
+
(quality_prior − latency − token cost) over the strategy tiers. The gate
|
|
29
|
+
makes ZERO generative-LLM calls — using an LLM to decide whether to call one
|
|
30
|
+
is circular (OP-33).
|
|
31
|
+
goal: |
|
|
32
|
+
Always-on retrieval empirically HURTS (SRACG AAAI 2026: −2.6 to −3.6pp);
|
|
33
|
+
selective retrieval helps (+2.4 to +7.1pp). `strategy=none` is not a fallback
|
|
34
|
+
— it is the correct answer for 30-40% of agent queries. This gate makes the
|
|
35
|
+
skip decision (and the which-signal decision) deterministically, before any
|
|
36
|
+
signal runs, so the retrieval bill is only paid when it pays off.
|
|
37
|
+
|
|
38
|
+
Deliverables:
|
|
39
|
+
1. complexity(query) -> float in [0,1] (CA-RAG: 0.6·wordlen/20 + 0.4·cues/3)
|
|
40
|
+
2. gate(query, *, entities=(), private_ids=frozenset(), tail_ids=frozenset(),
|
|
41
|
+
config=DEFAULT_GATE) -> RoutingDecision(strategy, stage, reason,
|
|
42
|
+
complexity, freshness_required, scores)
|
|
43
|
+
3. GateConfig — the calibratable weights/priors/marker sets (OP-33 defaults
|
|
44
|
+
baked in; the pattern says calibrate per corpus after OP-31 eval).
|
|
45
|
+
Entities arrive already resolved (canonical_ids, from cairn.resolve tier-1 —
|
|
46
|
+
the caller's fast path); private_ids / tail_ids are caller-supplied static
|
|
47
|
+
label sets built at ontology-authoring time (OP-35). Closed inputs, no magic.
|
|
48
|
+
success_metric: "All constraint tests pass under the project venv (python 3.11), stdlib-only, byte-stable; strategy=none reachable on a plain query; zero LLM/embedding calls."
|
|
49
|
+
stakeholders:
|
|
50
|
+
- "agents paying per-token for retrieval they may not need"
|
|
51
|
+
- "M3.6 retrieve() (the gate is its first stage)"
|
|
52
|
+
- "M3.4 fusion / M3.5 assembler (strategy selects which signals run)"
|
|
53
|
+
|
|
54
|
+
design_decisions:
|
|
55
|
+
strategy_tiers:
|
|
56
|
+
decision: "strategy ∈ {none, grep, semantic, graph, composite}. none == direct LLM (skip retrieval). This mirrors OP-25 paradigms and OP-33 stage-2 output."
|
|
57
|
+
entities_are_pre_resolved:
|
|
58
|
+
decision: "The gate takes canonical_ids (already resolved by the caller via cairn tier-1), not raw text to resolve. private_ids/tail_ids are caller-supplied frozensets."
|
|
59
|
+
rationale: "Keeps the gate fast + deterministic + decoupled; resolution is Primitive B's job, popularity labels are authoring-time artifacts (OP-35). No hidden work."
|
|
60
|
+
utility_defaults_are_calibratable:
|
|
61
|
+
decision: "OP-33 defaults are baked into GateConfig: w_Q=0.6, w_L=0.2, w_C=0.2; quality_prior none=0.52/grep=0.64/semantic=0.73/graph=0.82/composite=0.86; tail_boost=0.15. Complexity lifts heavier tiers via a per-tier complexity_gain. All live in GateConfig — tunable, not hard-coded in the logic."
|
|
62
|
+
rationale: "OP-33: 'treat as defaults, calibrate per corpus'. The formula is fixed; the numbers are configuration."
|
|
63
|
+
|
|
64
|
+
interface:
|
|
65
|
+
type: "library"
|
|
66
|
+
tools:
|
|
67
|
+
- name: "gate"
|
|
68
|
+
description: "gate(query, *, entities=(), private_ids=frozenset(), tail_ids=frozenset(), config=DEFAULT_GATE) -> RoutingDecision."
|
|
69
|
+
returns: "RoutingDecision(strategy, stage, reason, complexity, freshness_required, scores)"
|
|
70
|
+
test: "packages/cairn-retrieval/tests/unit/test_gate.py"
|
|
71
|
+
- name: "complexity"
|
|
72
|
+
description: "complexity(query) -> float in [0,1] per the CA-RAG formula."
|
|
73
|
+
returns: "float"
|
|
74
|
+
test: "packages/cairn-retrieval/tests/unit/test_gate.py::test_complexity_formula"
|
|
75
|
+
|
|
76
|
+
constraints:
|
|
77
|
+
- id: "C-GATE-1"
|
|
78
|
+
statement: "Stage 1 bypass — a temporal marker ('today','current','latest','as of','now','recent','this week') or a version/release marker in the query returns stage=1, freshness_required=True, and a retrieve strategy (graph if an entity is present, else semantic). No scoring runs."
|
|
79
|
+
assumption_status: "KNOWN"
|
|
80
|
+
test: "packages/cairn-retrieval/tests/unit/test_gate.py::test_stage1_temporal"
|
|
81
|
+
test_provenance: "human_authored"
|
|
82
|
+
|
|
83
|
+
- id: "C-GATE-2"
|
|
84
|
+
statement: "Stage 1 bypass — ≥2 distinct resolved entities (or a relational cue with ≥1 entity) returns stage=1, strategy='composite'. A single resolved entity in private_ids returns stage=1, strategy='graph'. Documented precedence: multi-entity/relational > temporal > private."
|
|
85
|
+
assumption_status: "KNOWN"
|
|
86
|
+
test: "packages/cairn-retrieval/tests/unit/test_gate.py::test_stage1_relational_and_private"
|
|
87
|
+
test_provenance: "human_authored"
|
|
88
|
+
|
|
89
|
+
- id: "C-GATE-3"
|
|
90
|
+
statement: "complexity(q) == clip(0.6·(wordlen/20) + 0.4·(cues/3), 0, 1) where wordlen is the whitespace token count and cues counts relational keywords (compare, why, connect, between, history of, what changed, path, related). Hand-computed golden values match."
|
|
91
|
+
assumption_status: "KNOWN"
|
|
92
|
+
test: "packages/cairn-retrieval/tests/unit/test_gate.py::test_complexity_formula"
|
|
93
|
+
test_provenance: "human_authored"
|
|
94
|
+
execution_oracle:
|
|
95
|
+
type: "golden_fixture"
|
|
96
|
+
note: "Two queries with hand-computed wordlen/cues -> exact float asserted."
|
|
97
|
+
|
|
98
|
+
- id: "C-GATE-4"
|
|
99
|
+
statement: "Stage 2 cost function — utility(s) = w_Q·(quality_prior[s] + c·complexity_gain[s] + tail_boost·is_retrieve(s,tail)) − w_L·latency_cost[s] − w_C·token_cost[s]; argmax over the tiers (cheapest tier wins ties). A plain popular-entity query yields strategy='none' (strategy=none REACHABLE); the SAME query with the entity in tail_ids flips to a retrieve tier. scores is populated with every tier's utility."
|
|
100
|
+
assumption_status: "KNOWN"
|
|
101
|
+
test: "packages/cairn-retrieval/tests/unit/test_gate.py::test_stage2_none_reachable_and_tail_flip"
|
|
102
|
+
test_provenance: "execution_derived"
|
|
103
|
+
execution_oracle:
|
|
104
|
+
type: "golden_fixture"
|
|
105
|
+
note: "Hand-computed: utility(none)=0.312 beats grep=0.3067 on a plain query; with tail_boost grep=0.3967 beats none. Argmax + tie-break asserted."
|
|
106
|
+
|
|
107
|
+
# CRITICAL, LAST — the OP-33 keystone: no LLM to decide whether to call an LLM.
|
|
108
|
+
- id: "C-GATE-5"
|
|
109
|
+
statement: "The gate makes ZERO generative-LLM and ZERO embedding calls and reads no wall clock; it is byte-stable (same inputs -> identical RoutingDecision) and imports nothing outside stdlib + cairn_retrieval. Freshness is decided by static markers, not by a clock."
|
|
110
|
+
assumption_status: "KNOWN"
|
|
111
|
+
conflict_prone: true
|
|
112
|
+
anti_pattern: "Calling a small LLM/classifier API to label the query's strategy 'because it's more accurate'. That is circular — it costs the very call the gate exists to avoid and injects non-determinism. The optional Adaptive-RAG classifier (OP-33) is a LOCAL trained model, never a generative-LLM API, and is out of scope for the default path."
|
|
113
|
+
test: "packages/cairn-retrieval/tests/unit/test_gate.py::test_no_llm_byte_stable"
|
|
114
|
+
test_provenance: "execution_derived"
|
|
115
|
+
execution_oracle:
|
|
116
|
+
type: "differential_test"
|
|
117
|
+
note: "Static-scan the module for llm/openai/anthropic/requests/clock imports (assert none); run gate twice on the same inputs and assert identical decisions."
|
|
118
|
+
|
|
119
|
+
constraint_satisfiability:
|
|
120
|
+
count: 5
|
|
121
|
+
joint_satisfaction_test: "packages/cairn-retrieval/tests/unit/test_gate.py::test_joint_gate"
|
|
122
|
+
joint_note: >
|
|
123
|
+
One test drives all stages on the same config: a temporal+entity query -> stage
|
|
124
|
+
1 graph + freshness (C-1), a two-entity query -> composite (C-2), a plain query
|
|
125
|
+
-> none via the utility argmax (C-3 complexity feeds C-4), the tail variant ->
|
|
126
|
+
grep (C-4), and asserts the whole run is byte-stable with no LLM (C-5).
|
|
127
|
+
conflicts_checked: >
|
|
128
|
+
C-1 (temporal -> retrieve) vs C-4 (none reachable): no conflict — temporal is a
|
|
129
|
+
Stage-1 bypass that never reaches Stage 2, so 'none' is only possible when no
|
|
130
|
+
freshness flag fired. C-5 (no clock) vs C-1 (freshness): consistent — freshness
|
|
131
|
+
is a STATIC marker match on the query text, not a wall-clock read.
|
|
132
|
+
|
|
133
|
+
non_goals:
|
|
134
|
+
- "Running any retrieval signal (the gate only DECIDES; grep/semantic/graph are M3.2-M3.4)."
|
|
135
|
+
- "Resolving entities from raw text (caller passes resolved canonical_ids)."
|
|
136
|
+
- "The optional Adaptive-RAG trained classifier (a local model swap for Stage 2; not the default path)."
|
|
137
|
+
- "Per-corpus calibration of the weights (defaults ship; OP-31 eval calibrates later)."
|
|
138
|
+
|
|
139
|
+
changelog:
|
|
140
|
+
- version: "0.1.0"
|
|
141
|
+
date: "2026-07-12"
|
|
142
|
+
change: "Initial intent for M3.1 adaptive gate. Two-stage LLM-free gate (deterministic bypass + CA-RAG utility), strategy=none reachable, tail-entity flip, complexity formula. 5 constraints, zero-LLM keystone (conflict_prone) last; joint test defined."
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# ==============================================================================
|
|
2
|
+
# cairn-retrieval — Module Intent: context assembler (M3.5)
|
|
3
|
+
# ==============================================================================
|
|
4
|
+
# License: MIT · Copyright (c) 2026 Leo Celis · https://ivdframework.dev
|
|
5
|
+
# Parent: ../../../cairn_system_intent.yaml (v0.2.1)
|
|
6
|
+
# Patterns: patterns_retrieval_knowledge.yaml — OP-29 (context assembler, PRIMARY),
|
|
7
|
+
# OP-15 (token budget), FF-11 (length alone degrades — fewest chunks wins)
|
|
8
|
+
# Reuses: cairn.cosine (diversity + dedup on PRECOMPUTED embeddings)
|
|
9
|
+
# ==============================================================================
|
|
10
|
+
|
|
11
|
+
scope:
|
|
12
|
+
level: "module"
|
|
13
|
+
type: "feature"
|
|
14
|
+
granularity: "fine"
|
|
15
|
+
|
|
16
|
+
metadata:
|
|
17
|
+
feature_id: "cairn_retrieval_assembler"
|
|
18
|
+
category: "agent_infrastructure"
|
|
19
|
+
tags: ["assembler", "mmr", "budget", "lost-in-the-middle", "trace", "deterministic"]
|
|
20
|
+
status: "in_progress"
|
|
21
|
+
introduced_version: "0.0.1"
|
|
22
|
+
|
|
23
|
+
intent:
|
|
24
|
+
summary: >
|
|
25
|
+
A pure-function pipeline that turns fused candidates into the smallest
|
|
26
|
+
sufficient context package: budget -> MMR-select -> cosine-dedup -> fold-order
|
|
27
|
+
-> emit + trace manifest (OP-29). Zero generative-LLM calls, no re-embedding
|
|
28
|
+
on the hot path (uses precomputed vectors), byte-stable.
|
|
29
|
+
goal: |
|
|
30
|
+
FF-11 (Du 2025): context LENGTH alone degrades accuracy 13.9-85% even with
|
|
31
|
+
perfect retrieval — fewest chunks wins. So after fusion the assembler builds
|
|
32
|
+
the MINIMAL package: relevant AND diverse (MMR), de-duplicated, and ordered so
|
|
33
|
+
the strongest evidence is at the edges (lost-in-the-middle, Liu 2023), with a
|
|
34
|
+
trace manifest for auditability.
|
|
35
|
+
|
|
36
|
+
Deliverables:
|
|
37
|
+
1. assemble(candidates, *, budget, lam=0.5, dedup_threshold=0.95)
|
|
38
|
+
-> AssembledContext(chunks, text, manifest, total_tokens).
|
|
39
|
+
Candidate = (doc_id, content, tokens, rel, embedding). Pipeline:
|
|
40
|
+
A budget — greedy pack within `budget` tokens (no knapsack).
|
|
41
|
+
B MMR-select — argmax λ·rel − (1−λ)·max cos(e_i, e_j∈S); tie rel desc,
|
|
42
|
+
then doc_id; pick highest-MMR that fits remaining budget.
|
|
43
|
+
C dedup — hard gate: skip if cos(e_i, any selected) ≥ dedup_threshold.
|
|
44
|
+
D fold-order — rel rank-1 -> FIRST, rank-2 -> LAST, rank-3 -> second, …
|
|
45
|
+
(weakest in the middle).
|
|
46
|
+
E emit — concatenated content with per-chunk provenance tags +
|
|
47
|
+
ManifestEntry(doc_id, tokens, rel, mmr_score, position).
|
|
48
|
+
success_metric: "All constraint tests pass under the project venv (python 3.11); same (candidates, budget, lam) -> byte-identical package; zero LLM, no re-embed (no embedder param)."
|
|
49
|
+
stakeholders:
|
|
50
|
+
- "M3.6 retrieve() (assembler is the final stage before the model sees context)"
|
|
51
|
+
- "agents that must stay under a token budget without losing the key evidence"
|
|
52
|
+
|
|
53
|
+
design_decisions:
|
|
54
|
+
precomputed_embeddings_only:
|
|
55
|
+
decision: "Candidates carry their embedding vector; the assembler NEVER embeds. It has no embedder parameter."
|
|
56
|
+
rationale: "OP-29: 'NEVER re-embed on hot path'. Embeddings were computed by the semantic signal at build; the assembler only reads them (cairn.cosine) for MMR diversity + dedup. Keeps the hot path pure + deterministic."
|
|
57
|
+
greedy_not_knapsack:
|
|
58
|
+
decision: "Greedy packing — iteratively pick the highest-MMR candidate that fits remaining budget. No optimal knapsack."
|
|
59
|
+
rationale: "OP-29: knapsack is non-deterministic on ties. Greedy + a fixed tie-break (rel desc, doc_id asc) is byte-stable, which the determinism invariant requires over marginal packing efficiency."
|
|
60
|
+
fold_ordering:
|
|
61
|
+
decision: "After selection, order by rel with a fold: strongest first, second-strongest last, alternating inward — weakest in the middle."
|
|
62
|
+
rationale: "Lost-in-the-middle (Liu 2023): a relevant doc in the middle drops accuracy 20-30pt. Edges are the high-attention positions."
|
|
63
|
+
|
|
64
|
+
interface:
|
|
65
|
+
type: "library"
|
|
66
|
+
tools:
|
|
67
|
+
- name: "assemble"
|
|
68
|
+
description: "assemble(candidates, *, budget, lam=0.5, dedup_threshold=0.95) -> AssembledContext."
|
|
69
|
+
returns: "AssembledContext(chunks, text, manifest, total_tokens)"
|
|
70
|
+
test: "packages/cairn-retrieval/tests/unit/test_assembler.py"
|
|
71
|
+
|
|
72
|
+
constraints:
|
|
73
|
+
- id: "C-ASM-1"
|
|
74
|
+
statement: "MMR + greedy budget: selection maximizes λ·rel − (1−λ)·max cos-to-selected; the total tokens of selected chunks never exceeds `budget`; a candidate that does not fit the remaining budget is skipped. With diversity weight, a lower-rel but DIVERSE candidate can be chosen over a higher-rel near-duplicate."
|
|
75
|
+
assumption_status: "KNOWN"
|
|
76
|
+
test: "packages/cairn-retrieval/tests/unit/test_assembler.py::test_mmr_budget"
|
|
77
|
+
test_provenance: "human_authored"
|
|
78
|
+
execution_oracle:
|
|
79
|
+
type: "golden_fixture"
|
|
80
|
+
note: "3 candidates, a≈b (same vector), c orthogonal; budget fits 2; MMR picks a then c (diverse) over b (near-dup). Hand-computed MMR values."
|
|
81
|
+
|
|
82
|
+
- id: "C-ASM-2"
|
|
83
|
+
statement: "Dedup hard gate: a candidate whose cosine to ANY already-selected chunk is >= dedup_threshold is dropped (the higher-rel one is kept), even if it fits the budget."
|
|
84
|
+
assumption_status: "KNOWN"
|
|
85
|
+
test: "packages/cairn-retrieval/tests/unit/test_assembler.py::test_dedup_hard_gate"
|
|
86
|
+
test_provenance: "human_authored"
|
|
87
|
+
|
|
88
|
+
- id: "C-ASM-3"
|
|
89
|
+
statement: "Fold ordering: selected chunks are ordered rel rank-1 -> position 0 (first), rank-2 -> last, rank-3 -> second, rank-4 -> second-to-last, … so the weakest evidence sits in the middle. Deterministic."
|
|
90
|
+
assumption_status: "KNOWN"
|
|
91
|
+
test: "packages/cairn-retrieval/tests/unit/test_assembler.py::test_fold_ordering"
|
|
92
|
+
test_provenance: "human_authored"
|
|
93
|
+
|
|
94
|
+
- id: "C-ASM-4"
|
|
95
|
+
statement: "Trace manifest + provenance: every selected chunk has a ManifestEntry(doc_id, tokens, rel, mmr_score, position) with position matching its place in the emitted order; the emitted text carries a per-chunk provenance tag naming the doc_id. total_tokens equals the sum of selected chunk tokens."
|
|
96
|
+
assumption_status: "KNOWN"
|
|
97
|
+
test: "packages/cairn-retrieval/tests/unit/test_assembler.py::test_manifest_and_provenance"
|
|
98
|
+
test_provenance: "human_authored"
|
|
99
|
+
|
|
100
|
+
# CRITICAL, LAST — zero-LLM, no re-embed, deterministic (OP-29 keystone).
|
|
101
|
+
- id: "C-ASM-5"
|
|
102
|
+
statement: "The assembler makes ZERO generative-LLM calls and NEVER embeds (no embedder parameter; it only reads precomputed candidate embeddings via cairn.cosine). It imports only stdlib + cairn, reads no wall clock, and is byte-stable: same (candidates, budget, lam, dedup_threshold) -> identical AssembledContext."
|
|
103
|
+
assumption_status: "KNOWN"
|
|
104
|
+
conflict_prone: true
|
|
105
|
+
anti_pattern: "Calling an LLM to 'summarize/compress' the selected chunks, or re-embedding candidates inside assemble 'to be safe'. Both break the pure-function + zero-hot-path-cost contract. The assembler only selects, orders, and emits from precomputed inputs."
|
|
106
|
+
test: "packages/cairn-retrieval/tests/unit/test_assembler.py::test_no_llm_no_reembed_byte_stable"
|
|
107
|
+
test_provenance: "execution_derived"
|
|
108
|
+
execution_oracle:
|
|
109
|
+
type: "differential_test"
|
|
110
|
+
note: "Static-scan for LLM/network/embedding/clock imports (assert none) + confirm assemble has no embedder param; assemble twice and assert identical package."
|
|
111
|
+
|
|
112
|
+
constraint_satisfiability:
|
|
113
|
+
count: 5
|
|
114
|
+
joint_satisfaction_test: "packages/cairn-retrieval/tests/unit/test_assembler.py::test_joint_assembler"
|
|
115
|
+
joint_note: >
|
|
116
|
+
One test assembles a candidate set that exercises budget capping + MMR
|
|
117
|
+
diversity (C-1), a dedup drop (C-2), fold ordering (C-3), a full manifest with
|
|
118
|
+
provenance (C-4), all zero-LLM + no re-embed + byte-stable (C-5).
|
|
119
|
+
conflicts_checked: >
|
|
120
|
+
C-1 (MMR diversity) vs C-2 (dedup): complementary — MMR softly demotes similar
|
|
121
|
+
docs, the dedup gate hard-drops near-identical ones. C-3 (fold by rel) vs C-1
|
|
122
|
+
(select by MMR): no conflict — selection is by MMR, final ORDER is by rel; two
|
|
123
|
+
separate stages. C-5 (no re-embed) vs C-1/C-2 (need cosine): consistent —
|
|
124
|
+
cosine runs on PRECOMPUTED candidate embeddings, no embedding is computed.
|
|
125
|
+
|
|
126
|
+
non_goals:
|
|
127
|
+
- "LLM summarization / compression of chunks (see C-ASM-5 — pure selection only)."
|
|
128
|
+
- "Re-embedding or embedding candidates (embeddings are precomputed inputs)."
|
|
129
|
+
- "Tokenization / token counting (the caller supplies each candidate's token count)."
|
|
130
|
+
- "Knapsack-optimal packing (greedy + fixed tie-break for determinism)."
|
|
131
|
+
|
|
132
|
+
changelog:
|
|
133
|
+
- version: "0.1.0"
|
|
134
|
+
date: "2026-07-12"
|
|
135
|
+
change: "Initial intent for M3.5 context assembler. Pure pipeline budget->MMR->dedup->fold-order->emit+trace over precomputed embeddings; zero-LLM, no re-embed. 5 constraints, zero-LLM/no-re-embed keystone (conflict_prone) last; joint test defined."
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# ==============================================================================
|
|
2
|
+
# cairn-retrieval — Module Intent: connection suggestion (cold-start link pred.)
|
|
3
|
+
# ==============================================================================
|
|
4
|
+
# License: MIT · Copyright (c) 2026 Leo Celis · https://ivdframework.dev
|
|
5
|
+
# Parent: ../../../cairn_system_intent.yaml (v0.2.1)
|
|
6
|
+
# Patterns: patterns_retrieval_knowledge.yaml OP-36 (connection suggestion),
|
|
7
|
+
# OP-33 (link_ranking — the node<->node measure this generalises).
|
|
8
|
+
# Research: docs/research/foundations/LINK_PREDICTION_AND_CONNECTION_SUGGESTION_RESEARCH_2026.md
|
|
9
|
+
# Feedback: docs/feedback/retrieval_ranking_feedback.yaml (round 1 boundaries honored)
|
|
10
|
+
# Boundary note: this is RANKING (relevance), which lives in cairn-retrieval —
|
|
11
|
+
# NOT in the entity engine, whose traverse() only does structural bounded
|
|
12
|
+
# closure. The engine supplies the graph; this weighs a CANDIDATE against it.
|
|
13
|
+
# ==============================================================================
|
|
14
|
+
|
|
15
|
+
scope:
|
|
16
|
+
level: "module"
|
|
17
|
+
type: "feature"
|
|
18
|
+
granularity: "fine"
|
|
19
|
+
|
|
20
|
+
metadata:
|
|
21
|
+
feature_id: "cairn_retrieval_connection_suggestion"
|
|
22
|
+
category: "agent_infrastructure"
|
|
23
|
+
tags: ["ranking", "link-prediction", "cold-start", "schema-matching", "deterministic"]
|
|
24
|
+
status: "in_progress"
|
|
25
|
+
introduced_version: "0.2.0"
|
|
26
|
+
|
|
27
|
+
intent:
|
|
28
|
+
summary: "Given a CANDIDATE node not yet in the graph (a table/module about to be created), rank the existing entities it should connect to. Cold-start: the candidate has no edges, so every structural index is 0; content similarity (TF-IDF COSINE over shared concepts, corpus IDF) is the primary signal. An optional Adamic-Adar/Resource-Allocation structural term fires only when the caller supplies seed entities the candidate provisionally relates to. Deterministic, zero-dep, zero-LLM on the default path."
|
|
29
|
+
goal: |
|
|
30
|
+
The candidate-vs-corpus generalisation of link_ranking (OP-33). Where
|
|
31
|
+
rank_links scores existing document<->document pairs, suggest_connections
|
|
32
|
+
scores a NEW candidate against every existing entity:
|
|
33
|
+
concept_idf(existing) -> corpus IDF (reused from link_ranking);
|
|
34
|
+
candidate tf*idf vector -> built against THAT corpus IDF;
|
|
35
|
+
content_score(y) -> cosine(candidate_vec, y_vec) in [0,1];
|
|
36
|
+
structural_score(y) -> RA/AA over seeds ∩ Γ(y) (0 without seeds);
|
|
37
|
+
total(y) -> content + graph_weight·structural_norm.
|
|
38
|
+
Returns ranked ConnectionSuggestion(target_id, score, content_score,
|
|
39
|
+
structural_score, shared, via) — provenance so a human/agent sees WHY.
|
|
40
|
+
success_metric: "A candidate sharing a rare concept with entity A and only a ubiquitous concept with B ranks A above B and drops B (score 0). With no seeds, structural_score is exactly 0 for all targets. A candidate whose only concepts are novel (absent from corpus) returns []. Byte-identical across runs."
|
|
41
|
+
stakeholders:
|
|
42
|
+
- "agents/developers designing a new table/module who need the existing-graph connections it should have (the schema-graph gap)"
|
|
43
|
+
|
|
44
|
+
interface:
|
|
45
|
+
type: "library"
|
|
46
|
+
tools:
|
|
47
|
+
- name: "suggest_connections"
|
|
48
|
+
description: "suggest_connections(candidate, existing, *, top_k=5, doc_type='document', concept_type='concept', seeds=(), graph_weight=0.25) -> list[ConnectionSuggestion]. Rank existing entities the candidate should link to."
|
|
49
|
+
parameters:
|
|
50
|
+
- name: "candidate"
|
|
51
|
+
type: "object"
|
|
52
|
+
required: true
|
|
53
|
+
description: "An Entity (NOT in `existing`) whose relations to concept-type entities describe it."
|
|
54
|
+
- name: "existing"
|
|
55
|
+
type: "array"
|
|
56
|
+
required: true
|
|
57
|
+
description: "The frozen graph's entities (e.g. table.canonical_entities())."
|
|
58
|
+
- name: "seeds"
|
|
59
|
+
type: "array"
|
|
60
|
+
required: false
|
|
61
|
+
description: "Optional canonical_ids the candidate provisionally relates to; enables the structural term."
|
|
62
|
+
returns: "list[ConnectionSuggestion(target_id, score, content_score, structural_score, shared, via)] — deterministic order"
|
|
63
|
+
test: "packages/cairn-retrieval/tests/unit/test_connections.py"
|
|
64
|
+
|
|
65
|
+
# Critical constraints LAST (position bias: later = weighted more).
|
|
66
|
+
constraints:
|
|
67
|
+
- name: "content_similarity_cold_start"
|
|
68
|
+
requirement: "content_score(candidate, y) = cosine of the candidate's TF-IDF concept vector against y's, in [0,1], using IDF computed from the EXISTING corpus (concept_idf). A concept the candidate has that no existing entity mentions (df=0 in corpus) contributes 0 — it matches nothing. This is the primary signal because a cold-start candidate has no edges for structural indices to use."
|
|
69
|
+
test: "packages/cairn-retrieval/tests/unit/test_connections.py::test_content_ranks_shared_rare_concept_first"
|
|
70
|
+
test_provenance: "execution_derived"
|
|
71
|
+
execution_oracle:
|
|
72
|
+
golden_fixture: "candidate shares rare concept with A (cosine 1.0), ubiquitous-only with B (IDF 0 -> dropped); A ranked, B absent"
|
|
73
|
+
assumption_status: "KNOWN"
|
|
74
|
+
entropy: "near-zero"
|
|
75
|
+
priority: 4
|
|
76
|
+
|
|
77
|
+
- name: "structural_signal_optional_seeds_only"
|
|
78
|
+
requirement: "structural_score(y) = sum over s in (seeds ∩ Γ(y)) of 1/deg(s) (Resource Allocation; AA=1/log deg is the documented alternative), where Γ and deg are over the entity-entity graph among `existing`. When seeds is empty, structural_score(y) = 0 for EVERY y — the API must not manufacture structural evidence a cold node cannot have. Seeds themselves are inputs, not results."
|
|
79
|
+
test: "packages/cairn-retrieval/tests/unit/test_connections.py::test_structural_zero_without_seeds"
|
|
80
|
+
test_provenance: "execution_derived"
|
|
81
|
+
execution_oracle:
|
|
82
|
+
golden_fixture: "no seeds -> every structural_score == 0.0; with seed S, a target sharing S as neighbour gets 1/deg(S)"
|
|
83
|
+
assumption_status: "KNOWN"
|
|
84
|
+
entropy: "near-zero"
|
|
85
|
+
priority: 3
|
|
86
|
+
|
|
87
|
+
- name: "provenance_and_closed_world"
|
|
88
|
+
requirement: "Every suggestion carries its evidence: `shared` = the shared concept ids that earned content_score, `via` = the seed ids that contributed structural_score. A candidate with empty concepts and no seeds, or with zero overlap, returns [] — never a fabricated connection. Suggestions with total score 0 are dropped."
|
|
89
|
+
test: "packages/cairn-retrieval/tests/unit/test_connections.py::test_provenance_and_empty_on_no_overlap"
|
|
90
|
+
test_provenance: "execution_derived"
|
|
91
|
+
execution_oracle:
|
|
92
|
+
golden_fixture: "no overlap -> []; a real suggestion -> shared non-empty; seed-driven -> via non-empty"
|
|
93
|
+
assumption_status: "KNOWN"
|
|
94
|
+
entropy: "near-zero"
|
|
95
|
+
priority: 2
|
|
96
|
+
|
|
97
|
+
- name: "deterministic_pure_graph_no_engine_leak"
|
|
98
|
+
requirement: "Output order deterministic: total desc then target_id asc; only real existing ids appear (never the candidate, never a synthesized id). Pure stdlib (math only), no LLM, no network, no third-party deps; same inputs -> byte-identical result. Lives in cairn-retrieval; does not modify the engine or its traverse()."
|
|
99
|
+
test: "packages/cairn-retrieval/tests/unit/test_connections.py::test_deterministic_and_closed_world"
|
|
100
|
+
test_provenance: "execution_derived"
|
|
101
|
+
execution_oracle:
|
|
102
|
+
property_test: "repeated calls repr-equal; every target_id exists in `existing` and != candidate id; module imports stdlib + cairn_engine only"
|
|
103
|
+
assumption_status: "KNOWN"
|
|
104
|
+
entropy: "near-zero"
|
|
105
|
+
priority: 1
|
|
106
|
+
|
|
107
|
+
constraint_satisfiability:
|
|
108
|
+
conflicts_checked: true
|
|
109
|
+
known_tensions:
|
|
110
|
+
- tension: "content vs structural scale mismatch (cosine in [0,1] vs sum of 1/deg)"
|
|
111
|
+
resolution: "structural_norm = structural_score / len(seeds) (≤1); fused content-dominant via graph_weight=0.25 default."
|
|
112
|
+
simultaneous_satisfaction: "All satisfiable: one pure function; content and structural are additive terms, provenance is carried alongside, determinism from sorted output + rounding."
|
|
113
|
+
joint_satisfaction_test: "packages/cairn-retrieval/tests/unit/test_connections.py::test_joint_connection_suggestion"
|
|
114
|
+
|
|
115
|
+
rationale:
|
|
116
|
+
decision: "Content-first (TF-IDF cosine, corpus IDF) with an optional seed-gated RA/AA structural term; scoring lives in cairn-retrieval, generalising link_ranking to a candidate."
|
|
117
|
+
reasoning: "OP-36. A brand-new node has no edges, so structural indices (CN/AA/RA) are 0 — the cold-start regime, where the literature is consistent that content/attribute similarity is the signal (Lü & Zhou; Rahm & Bernstein linguistic matcher; cold-start link-prediction papers). Reusing concept_idf keeps one relevance mechanism, not two. The structural term is honestly gated on seeds because it is meaningless without at least one provisional edge."
|
|
118
|
+
evidence: "docs/research/foundations/LINK_PREDICTION_AND_CONNECTION_SUGGESTION_RESEARCH_2026.md; OP-36; feedback round 1 boundaries."
|
|
119
|
+
date: "2026-07-14"
|
|
120
|
+
stakeholder: "Maintainer"
|
|
121
|
+
|
|
122
|
+
alternatives:
|
|
123
|
+
- name: "Matrix factorisation / graph autoencoder for cold-start"
|
|
124
|
+
rejected_because: "Learned, non-deterministic, dependency-heavy — violates zero-dep/byte-stable/zero-LLM default. Documented as an opt-in accelerator, never the default path."
|
|
125
|
+
date_tested: "2026-07-14"
|
|
126
|
+
- name: "Structural indices as the primary signal"
|
|
127
|
+
rejected_because: "A cold-start candidate has NO edges — CN/AA/RA are all 0. Structural cannot be primary for a node that does not yet exist; content must lead."
|
|
128
|
+
date_tested: "2026-07-14"
|
|
129
|
+
- name: "Put candidate scoring in the engine's traverse()"
|
|
130
|
+
rejected_because: "traverse() does structural bounded closure over an EXISTING node; a not-yet-existing candidate scored by relevance is retrieval's job (OP-34 boundary, feedback round 1)."
|
|
131
|
+
date_tested: "2026-07-14"
|
|
132
|
+
|
|
133
|
+
risks:
|
|
134
|
+
- condition: "Caller passes a candidate whose concepts are all novel to the corpus."
|
|
135
|
+
action: "Returns [] (closed world) — correct: nothing to connect to yet. Caller may author the concepts first, then re-run."
|
|
136
|
+
monitor: "n/a"
|
|
137
|
+
severity: "low"
|
|
138
|
+
- condition: "graph_weight mis-set so structural swamps content."
|
|
139
|
+
action: "Default 0.25 keeps content dominant; documented as calibrate-per-corpus, not a silent default change."
|
|
140
|
+
monitor: "example output review"
|
|
141
|
+
severity: "low"
|
|
142
|
+
|
|
143
|
+
implementation:
|
|
144
|
+
current: "authored alongside this intent"
|
|
145
|
+
version: 1
|
|
146
|
+
tests: "packages/cairn-retrieval/tests/unit/test_connections.py"
|
|
147
|
+
documentation: "module docstrings; ROADMAP; example examples/schema_connection_suggestion.py"
|
|
148
|
+
|
|
149
|
+
verification:
|
|
150
|
+
status: "tests authored with implementation; hand-computed cosine + RA golden values"
|
|
151
|
+
test_cases:
|
|
152
|
+
- name: "joint"
|
|
153
|
+
description: "One suggest_connections call satisfying all four constraints on a hand-built graph"
|
|
154
|
+
input: {existing: "3 entities + concepts, one rare one ubiquitous", candidate: "shares rare with A", seeds: "one"}
|
|
155
|
+
expected_output: {A_ranked_first: true, ubiquitous_only_dropped: true, structural_zero_without_seeds: true, provenance_present: true, deterministic: true}
|
|
156
|
+
validates_constraint: "ALL (joint)"
|
|
157
|
+
|
|
158
|
+
changelog:
|
|
159
|
+
- version: "0.1.0"
|
|
160
|
+
date: "2026-07-14"
|
|
161
|
+
change: "Initial intent for connection suggestion (cold-start link prediction): 4 constraints, critical-last, joint-satisfaction test. Content-first TF-IDF cosine reusing concept_idf; optional seed-gated RA/AA structural term; provenance + closed-world; ranking kept out of the engine."
|
|
162
|
+
reason: "IVD Rule 1 — intent before implementation; the schema-graph gap (a new table with no map of existing tables) surfaced the concrete need. Follows the development loop: research -> feedback -> pattern (OP-36) -> implement."
|