maggraph 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {maggraph-0.3.0 → maggraph-0.4.0}/Cargo.lock +2 -2
- {maggraph-0.3.0 → maggraph-0.4.0}/Cargo.toml +1 -1
- {maggraph-0.3.0 → maggraph-0.4.0}/PKG-INFO +3 -1
- {maggraph-0.3.0 → maggraph-0.4.0}/README.md +2 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/__init__.pyi +44 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/benches/index_scale.rs +17 -3
- maggraph-0.4.0/maggraph/src/batch.rs +379 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/index.rs +10 -1
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/lib.rs +12 -2
- maggraph-0.4.0/maggraph/src/memory.rs +143 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/python/mod.rs +153 -5
- maggraph-0.4.0/maggraph/src/query.rs +628 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/pyproject.toml +1 -1
- maggraph-0.3.0/maggraph/src/memory.rs +0 -66
- maggraph-0.3.0/maggraph/src/query.rs +0 -235
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/Cargo.toml +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/__init__.py +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/benches/traversal.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/py.typed +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/agent/mod.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/config/mod.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/config/schema.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/error.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/graph.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/lakehouse/cache.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/lakehouse/content.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/lakehouse/mod.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/lakehouse/resolver.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/lakehouse/uri.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/node/frontmatter.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/node/mod.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/recall.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/security.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/sync/engine.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/sync/lock.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/sync/mod.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/sync/policy.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/sync/repo.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/ui/api.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/ui/assets/app.js +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/ui/assets/index.html +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/ui/assets/style.css +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/ui/mod.rs +0 -0
- {maggraph-0.3.0 → maggraph-0.4.0}/maggraph/src/wikilink.rs +0 -0
|
@@ -839,7 +839,7 @@ checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f"
|
|
|
839
839
|
|
|
840
840
|
[[package]]
|
|
841
841
|
name = "maggraph"
|
|
842
|
-
version = "0.
|
|
842
|
+
version = "0.4.0"
|
|
843
843
|
dependencies = [
|
|
844
844
|
"axum",
|
|
845
845
|
"chrono",
|
|
@@ -862,7 +862,7 @@ dependencies = [
|
|
|
862
862
|
|
|
863
863
|
[[package]]
|
|
864
864
|
name = "maggraph-cli"
|
|
865
|
-
version = "0.
|
|
865
|
+
version = "0.4.0"
|
|
866
866
|
dependencies = [
|
|
867
867
|
"assert_cmd",
|
|
868
868
|
"axum",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: maggraph
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -205,6 +205,7 @@ maggraph sync pull
|
|
|
205
205
|
| `GraphIndex.list_nodes()` | All node ids (sorted) |
|
|
206
206
|
| `GraphIndex.read_node(id)` | `Node` with metadata + body |
|
|
207
207
|
| `GraphIndex.search(...)` | Structured search over ids, types, tags, frontmatter, links, body, and recency |
|
|
208
|
+
| `GraphIndex.hybrid_search(...)` | Explainable lexical + graph + recency + optional semantic retrieval with temporal/project filters |
|
|
208
209
|
| `GraphIndex.backlinks(id)` | Node ids that link to `id` |
|
|
209
210
|
| `GraphIndex.changed_since(unix)` | Files modified after a Unix timestamp |
|
|
210
211
|
| `GraphIndex.update_file(path)` | Refresh one changed markdown file in the index |
|
|
@@ -214,6 +215,7 @@ maggraph sync pull
|
|
|
214
215
|
| `GraphIndex.traverse_async(...)` | Async version |
|
|
215
216
|
| `GraphIndex.create_node(...)` | Write new node to disk + index |
|
|
216
217
|
| `GraphIndex.create_memory_node(...)` | Create typed memory nodes (`preference`, `project_fact`, `decision`, `task`, `session_summary`, `bookmark`, `tool_failure`) |
|
|
218
|
+
| `GraphIndex.apply_memory_batch(...)` | Preview or apply reviewed update/suppress/unsuppress/merge operations as one rollback-capable batch |
|
|
217
219
|
| `GraphIndex.update_node(id, body)` | Update body on disk |
|
|
218
220
|
| `GraphIndex.delete_node(id)` | Delete node from disk + index |
|
|
219
221
|
| `GraphIndex.suppress_node(id)` / `unsuppress_node(id)` | Mark/unmark stale or duplicate nodes |
|
|
@@ -168,6 +168,7 @@ maggraph sync pull
|
|
|
168
168
|
| `GraphIndex.list_nodes()` | All node ids (sorted) |
|
|
169
169
|
| `GraphIndex.read_node(id)` | `Node` with metadata + body |
|
|
170
170
|
| `GraphIndex.search(...)` | Structured search over ids, types, tags, frontmatter, links, body, and recency |
|
|
171
|
+
| `GraphIndex.hybrid_search(...)` | Explainable lexical + graph + recency + optional semantic retrieval with temporal/project filters |
|
|
171
172
|
| `GraphIndex.backlinks(id)` | Node ids that link to `id` |
|
|
172
173
|
| `GraphIndex.changed_since(unix)` | Files modified after a Unix timestamp |
|
|
173
174
|
| `GraphIndex.update_file(path)` | Refresh one changed markdown file in the index |
|
|
@@ -177,6 +178,7 @@ maggraph sync pull
|
|
|
177
178
|
| `GraphIndex.traverse_async(...)` | Async version |
|
|
178
179
|
| `GraphIndex.create_node(...)` | Write new node to disk + index |
|
|
179
180
|
| `GraphIndex.create_memory_node(...)` | Create typed memory nodes (`preference`, `project_fact`, `decision`, `task`, `session_summary`, `bookmark`, `tool_failure`) |
|
|
181
|
+
| `GraphIndex.apply_memory_batch(...)` | Preview or apply reviewed update/suppress/unsuppress/merge operations as one rollback-capable batch |
|
|
180
182
|
| `GraphIndex.update_node(id, body)` | Update body on disk |
|
|
181
183
|
| `GraphIndex.delete_node(id)` | Delete node from disk + index |
|
|
182
184
|
| `GraphIndex.suppress_node(id)` / `unsuppress_node(id)` | Mark/unmark stale or duplicate nodes |
|
|
@@ -9,6 +9,17 @@ class SearchResult(TypedDict):
|
|
|
9
9
|
summary: str
|
|
10
10
|
modified_unix: Optional[int]
|
|
11
11
|
|
|
12
|
+
class HybridSearchResult(TypedDict):
|
|
13
|
+
id: str
|
|
14
|
+
type: str
|
|
15
|
+
relative_path: str
|
|
16
|
+
score: float
|
|
17
|
+
signals: dict[str, float]
|
|
18
|
+
reasons: list[str]
|
|
19
|
+
summary: str
|
|
20
|
+
modified_unix: Optional[int]
|
|
21
|
+
canonical_id: str
|
|
22
|
+
|
|
12
23
|
class GraphChange(TypedDict):
|
|
13
24
|
id: str
|
|
14
25
|
relative_path: str
|
|
@@ -25,6 +36,12 @@ class RecallBundle(TypedDict):
|
|
|
25
36
|
relevance_reason: str
|
|
26
37
|
markdown: str
|
|
27
38
|
|
|
39
|
+
class MemoryBatchResult(TypedDict, total=False):
|
|
40
|
+
ok: bool
|
|
41
|
+
preview: bool
|
|
42
|
+
operations: int
|
|
43
|
+
applied: list[str]
|
|
44
|
+
|
|
28
45
|
class MagGraphError(Exception): ...
|
|
29
46
|
|
|
30
47
|
class ResolvedConfig:
|
|
@@ -54,6 +71,20 @@ class GraphIndex:
|
|
|
54
71
|
limit: int = 50,
|
|
55
72
|
modified_since_unix: int | None = None,
|
|
56
73
|
) -> list[SearchResult]: ...
|
|
74
|
+
def hybrid_search(
|
|
75
|
+
self,
|
|
76
|
+
query: str = "",
|
|
77
|
+
node_types: list[str] | None = None,
|
|
78
|
+
tags: list[str] | None = None,
|
|
79
|
+
project: str | None = None,
|
|
80
|
+
seed_ids: list[str] | None = None,
|
|
81
|
+
semantic_scores: dict[str, float] | None = None,
|
|
82
|
+
include_suppressed: bool = False,
|
|
83
|
+
include_superseded: bool = False,
|
|
84
|
+
limit: int = 20,
|
|
85
|
+
as_of_unix: int | None = None,
|
|
86
|
+
recency_half_life_days: float = 30.0,
|
|
87
|
+
) -> list[HybridSearchResult]: ...
|
|
57
88
|
def backlinks(self, node_id: str) -> list[str]: ...
|
|
58
89
|
def changed_since(self, since_unix: int) -> list[GraphChange]: ...
|
|
59
90
|
def update_file(self, path: str) -> Optional[str]: ...
|
|
@@ -79,12 +110,25 @@ class GraphIndex:
|
|
|
79
110
|
],
|
|
80
111
|
body: str = "",
|
|
81
112
|
links: list[str] | None = None,
|
|
113
|
+
project: str | None = None,
|
|
114
|
+
source_task: str | None = None,
|
|
115
|
+
source_session: str | None = None,
|
|
116
|
+
source_tool: str | None = None,
|
|
117
|
+
extraction_method: str | None = None,
|
|
118
|
+
confidence: float | None = None,
|
|
119
|
+
valid_from: str | None = None,
|
|
120
|
+
valid_until: str | None = None,
|
|
121
|
+
supersedes: str | None = None,
|
|
122
|
+
canonical_id: str | None = None,
|
|
82
123
|
) -> Node: ...
|
|
83
124
|
def update_node(self, node_id: str, body: str) -> None: ...
|
|
84
125
|
def delete_node(self, node_id: str) -> None: ...
|
|
85
126
|
def suppress_node(self, node_id: str, reason: str | None = None) -> None: ...
|
|
86
127
|
def unsuppress_node(self, node_id: str) -> None: ...
|
|
87
128
|
def merge_nodes(self, target_id: str, source_id: str) -> None: ...
|
|
129
|
+
def apply_memory_batch(
|
|
130
|
+
self, operations: list[dict[str, str]], preview: bool = False
|
|
131
|
+
) -> MemoryBatchResult: ...
|
|
88
132
|
def recall_bundle(
|
|
89
133
|
self, node_id: str, reason: str = "", body_chars: int = 1200
|
|
90
134
|
) -> RecallBundle: ...
|
|
@@ -2,7 +2,9 @@ use std::env;
|
|
|
2
2
|
use std::fs;
|
|
3
3
|
use std::time::{Duration, Instant};
|
|
4
4
|
|
|
5
|
-
use
|
|
5
|
+
use std::collections::HashMap;
|
|
6
|
+
|
|
7
|
+
use maggraph::{GraphIndex, HybridQueryOptions, QueryOptions};
|
|
6
8
|
use tempfile::TempDir;
|
|
7
9
|
|
|
8
10
|
fn sizes() -> Vec<usize> {
|
|
@@ -44,7 +46,7 @@ fn timed<T>(operation: impl FnOnce() -> T) -> (T, Duration) {
|
|
|
44
46
|
|
|
45
47
|
fn main() {
|
|
46
48
|
println!("MagGraph index scale benchmark");
|
|
47
|
-
println!("nodes,open_ms,search_ms,backlinks_ms,recall_us,update_us");
|
|
49
|
+
println!("nodes,open_ms,search_ms,hybrid_ms,backlinks_ms,recall_us,update_us");
|
|
48
50
|
|
|
49
51
|
for count in sizes() {
|
|
50
52
|
let temp = TempDir::new().expect("temp dir");
|
|
@@ -60,6 +62,17 @@ fn main() {
|
|
|
60
62
|
};
|
|
61
63
|
let (results, search) = timed(|| index.search(&options).expect("search"));
|
|
62
64
|
assert!(!results.is_empty());
|
|
65
|
+
let hybrid_options = HybridQueryOptions {
|
|
66
|
+
text: Some("needle group 42".into()),
|
|
67
|
+
tags: vec!["benchmark".into()],
|
|
68
|
+
seed_ids: vec!["node_000042".into()],
|
|
69
|
+
semantic_scores: HashMap::from([("node_000142".into(), 0.9)]),
|
|
70
|
+
limit: 50,
|
|
71
|
+
..HybridQueryOptions::default()
|
|
72
|
+
};
|
|
73
|
+
let (hybrid_results, hybrid) =
|
|
74
|
+
timed(|| index.hybrid_search(&hybrid_options).expect("hybrid search"));
|
|
75
|
+
assert!(!hybrid_results.is_empty());
|
|
63
76
|
|
|
64
77
|
let (_, backlinks) = timed(|| index.backlinks("node_000042").expect("backlinks"));
|
|
65
78
|
let (_, recall) = timed(|| {
|
|
@@ -74,9 +87,10 @@ fn main() {
|
|
|
74
87
|
});
|
|
75
88
|
|
|
76
89
|
println!(
|
|
77
|
-
"{count},{:.3},{:.3},{:.3},{:.3},{:.3}",
|
|
90
|
+
"{count},{:.3},{:.3},{:.3},{:.3},{:.3},{:.3}",
|
|
78
91
|
open.as_secs_f64() * 1_000.0,
|
|
79
92
|
search.as_secs_f64() * 1_000.0,
|
|
93
|
+
hybrid.as_secs_f64() * 1_000.0,
|
|
80
94
|
backlinks.as_secs_f64() * 1_000.0,
|
|
81
95
|
recall.as_secs_f64() * 1_000_000.0,
|
|
82
96
|
update.as_secs_f64() * 1_000_000.0,
|
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
//! Transaction-like reviewed memory operations with rollback on failure.
|
|
2
|
+
|
|
3
|
+
use std::collections::{BTreeMap, BTreeSet};
|
|
4
|
+
use std::fs::{self, File};
|
|
5
|
+
use std::io::Write;
|
|
6
|
+
use std::path::{Path, PathBuf};
|
|
7
|
+
|
|
8
|
+
use serde::{Deserialize, Serialize};
|
|
9
|
+
use uuid::Uuid;
|
|
10
|
+
|
|
11
|
+
use crate::error::{MagGraphError, Result};
|
|
12
|
+
use crate::index::GraphIndex;
|
|
13
|
+
use crate::node::Node;
|
|
14
|
+
|
|
15
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
16
|
+
pub enum MemoryBatchOperation {
|
|
17
|
+
UpdateBody {
|
|
18
|
+
id: String,
|
|
19
|
+
body: String,
|
|
20
|
+
},
|
|
21
|
+
Suppress {
|
|
22
|
+
id: String,
|
|
23
|
+
reason: Option<String>,
|
|
24
|
+
},
|
|
25
|
+
Unsuppress {
|
|
26
|
+
id: String,
|
|
27
|
+
},
|
|
28
|
+
Merge {
|
|
29
|
+
target_id: String,
|
|
30
|
+
source_id: String,
|
|
31
|
+
},
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
35
|
+
pub struct MemoryBatchResult {
|
|
36
|
+
pub applied: Vec<String>,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const JOURNAL_VERSION: u8 = 1;
|
|
40
|
+
const TRANSACTIONS_DIR: &str = "transactions";
|
|
41
|
+
|
|
42
|
+
#[derive(Debug, Serialize, Deserialize)]
|
|
43
|
+
struct BatchJournal {
|
|
44
|
+
version: u8,
|
|
45
|
+
snapshots: Vec<JournalSnapshot>,
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
#[derive(Debug, Serialize, Deserialize)]
|
|
49
|
+
struct JournalSnapshot {
|
|
50
|
+
relative_path: PathBuf,
|
|
51
|
+
backup: PathBuf,
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
pub fn apply_memory_batch(
|
|
55
|
+
index: &mut GraphIndex,
|
|
56
|
+
operations: &[MemoryBatchOperation],
|
|
57
|
+
) -> Result<MemoryBatchResult> {
|
|
58
|
+
validate_memory_batch(index, operations)?;
|
|
59
|
+
let snapshots = snapshots(index, operations)?;
|
|
60
|
+
let journal = prepare_journal(index.root_path(), &snapshots)?;
|
|
61
|
+
let mut applied = Vec::new();
|
|
62
|
+
for operation in operations {
|
|
63
|
+
let result = apply_one(index, operation);
|
|
64
|
+
match result {
|
|
65
|
+
Ok(label) => applied.push(label),
|
|
66
|
+
Err(error) => {
|
|
67
|
+
restore_journal(index.root_path(), &journal)?;
|
|
68
|
+
index.rescan()?;
|
|
69
|
+
return Err(error);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
fs::remove_dir_all(&journal).map_err(|error| {
|
|
74
|
+
MagGraphError::Index(format!("failed to remove completed batch journal: {error}"))
|
|
75
|
+
})?;
|
|
76
|
+
Ok(MemoryBatchResult { applied })
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/// Restore any prepared batch that did not reach journal cleanup before shutdown.
|
|
80
|
+
pub(crate) fn recover_incomplete_batches(root: &Path) -> Result<usize> {
|
|
81
|
+
let transactions = transaction_root(root);
|
|
82
|
+
if !transactions.exists() {
|
|
83
|
+
return Ok(0);
|
|
84
|
+
}
|
|
85
|
+
let mut recovered = 0;
|
|
86
|
+
for entry in fs::read_dir(&transactions).map_err(|error| {
|
|
87
|
+
MagGraphError::Index(format!("failed to inspect batch journals: {error}"))
|
|
88
|
+
})? {
|
|
89
|
+
let path = entry
|
|
90
|
+
.map_err(|error| {
|
|
91
|
+
MagGraphError::Index(format!("failed to read batch journal: {error}"))
|
|
92
|
+
})?
|
|
93
|
+
.path();
|
|
94
|
+
if !path.is_dir() {
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
if path.join("manifest.toml").exists() {
|
|
98
|
+
restore_journal(root, &path)?;
|
|
99
|
+
recovered += 1;
|
|
100
|
+
} else {
|
|
101
|
+
fs::remove_dir_all(&path).map_err(|error| {
|
|
102
|
+
MagGraphError::Index(format!(
|
|
103
|
+
"failed to remove unprepared batch journal: {error}"
|
|
104
|
+
))
|
|
105
|
+
})?;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
Ok(recovered)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
pub fn validate_memory_batch(
|
|
112
|
+
index: &GraphIndex,
|
|
113
|
+
operations: &[MemoryBatchOperation],
|
|
114
|
+
) -> Result<()> {
|
|
115
|
+
let mut removed = BTreeSet::new();
|
|
116
|
+
for operation in operations {
|
|
117
|
+
match operation {
|
|
118
|
+
MemoryBatchOperation::UpdateBody { id, .. }
|
|
119
|
+
| MemoryBatchOperation::Suppress { id, .. }
|
|
120
|
+
| MemoryBatchOperation::Unsuppress { id } => {
|
|
121
|
+
require_available(index, id, &removed)?;
|
|
122
|
+
}
|
|
123
|
+
MemoryBatchOperation::Merge {
|
|
124
|
+
target_id,
|
|
125
|
+
source_id,
|
|
126
|
+
} => {
|
|
127
|
+
if target_id == source_id {
|
|
128
|
+
return Err(MagGraphError::Index(
|
|
129
|
+
"cannot merge a node into itself".to_string(),
|
|
130
|
+
));
|
|
131
|
+
}
|
|
132
|
+
require_available(index, target_id, &removed)?;
|
|
133
|
+
require_available(index, source_id, &removed)?;
|
|
134
|
+
removed.insert(source_id.clone());
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
Ok(())
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
fn require_available(index: &GraphIndex, id: &str, removed: &BTreeSet<String>) -> Result<()> {
|
|
142
|
+
if removed.contains(id) || !index.contains(id) {
|
|
143
|
+
return Err(MagGraphError::NodeNotFound { id: id.to_string() });
|
|
144
|
+
}
|
|
145
|
+
Ok(())
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
fn snapshots(
|
|
149
|
+
index: &GraphIndex,
|
|
150
|
+
operations: &[MemoryBatchOperation],
|
|
151
|
+
) -> Result<BTreeMap<String, Node>> {
|
|
152
|
+
let mut ids = BTreeSet::new();
|
|
153
|
+
for operation in operations {
|
|
154
|
+
match operation {
|
|
155
|
+
MemoryBatchOperation::UpdateBody { id, .. }
|
|
156
|
+
| MemoryBatchOperation::Suppress { id, .. }
|
|
157
|
+
| MemoryBatchOperation::Unsuppress { id } => {
|
|
158
|
+
ids.insert(id.clone());
|
|
159
|
+
}
|
|
160
|
+
MemoryBatchOperation::Merge {
|
|
161
|
+
target_id,
|
|
162
|
+
source_id,
|
|
163
|
+
} => {
|
|
164
|
+
ids.insert(target_id.clone());
|
|
165
|
+
ids.insert(source_id.clone());
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
ids.into_iter()
|
|
170
|
+
.map(|id| index.read_node(&id).map(|node| (id, node)))
|
|
171
|
+
.collect()
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
fn apply_one(index: &mut GraphIndex, operation: &MemoryBatchOperation) -> Result<String> {
|
|
175
|
+
match operation {
|
|
176
|
+
MemoryBatchOperation::UpdateBody { id, body } => {
|
|
177
|
+
let mut node = index.read_node(id)?;
|
|
178
|
+
node.body = body.clone();
|
|
179
|
+
index.update_node(node)?;
|
|
180
|
+
Ok(format!("update:{id}"))
|
|
181
|
+
}
|
|
182
|
+
MemoryBatchOperation::Suppress { id, reason } => {
|
|
183
|
+
index.suppress_node(id, reason.as_deref())?;
|
|
184
|
+
Ok(format!("suppress:{id}"))
|
|
185
|
+
}
|
|
186
|
+
MemoryBatchOperation::Unsuppress { id } => {
|
|
187
|
+
index.unsuppress_node(id)?;
|
|
188
|
+
Ok(format!("unsuppress:{id}"))
|
|
189
|
+
}
|
|
190
|
+
MemoryBatchOperation::Merge {
|
|
191
|
+
target_id,
|
|
192
|
+
source_id,
|
|
193
|
+
} => {
|
|
194
|
+
index.merge_nodes(target_id, source_id)?;
|
|
195
|
+
Ok(format!("merge:{source_id}->{target_id}"))
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
fn transaction_root(root: &Path) -> PathBuf {
|
|
201
|
+
root.join(crate::config::METADATA_DIR_NAME)
|
|
202
|
+
.join(TRANSACTIONS_DIR)
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
fn prepare_journal(root: &Path, snapshots: &BTreeMap<String, Node>) -> Result<PathBuf> {
|
|
206
|
+
let journal_dir = transaction_root(root).join(Uuid::new_v4().to_string());
|
|
207
|
+
let backup_dir = journal_dir.join("backups");
|
|
208
|
+
fs::create_dir_all(&backup_dir).map_err(|error| {
|
|
209
|
+
MagGraphError::Index(format!("failed to create batch journal: {error}"))
|
|
210
|
+
})?;
|
|
211
|
+
let mut journal = BatchJournal {
|
|
212
|
+
version: JOURNAL_VERSION,
|
|
213
|
+
snapshots: Vec::new(),
|
|
214
|
+
};
|
|
215
|
+
for (index, node) in snapshots.values().enumerate() {
|
|
216
|
+
crate::security::validate_relative_node_path(&node.relative_path)?;
|
|
217
|
+
let backup = PathBuf::from("backups").join(format!("{index:04}.md"));
|
|
218
|
+
sync_write(&journal_dir.join(&backup), node.to_markdown()?.as_bytes())?;
|
|
219
|
+
journal.snapshots.push(JournalSnapshot {
|
|
220
|
+
relative_path: node.relative_path.clone(),
|
|
221
|
+
backup,
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
let manifest = toml::to_string(&journal).map_err(|error| {
|
|
225
|
+
MagGraphError::Index(format!("failed to serialize batch journal: {error}"))
|
|
226
|
+
})?;
|
|
227
|
+
sync_write(&journal_dir.join("manifest.toml"), manifest.as_bytes())?;
|
|
228
|
+
Ok(journal_dir)
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
fn restore_journal(root: &Path, journal_dir: &Path) -> Result<()> {
|
|
232
|
+
let manifest_path = journal_dir.join("manifest.toml");
|
|
233
|
+
let manifest = fs::read_to_string(&manifest_path).map_err(|error| {
|
|
234
|
+
MagGraphError::Index(format!(
|
|
235
|
+
"failed to read batch journal {}: {error}",
|
|
236
|
+
manifest_path.display()
|
|
237
|
+
))
|
|
238
|
+
})?;
|
|
239
|
+
let journal: BatchJournal = toml::from_str(&manifest)
|
|
240
|
+
.map_err(|error| MagGraphError::Index(format!("failed to parse batch journal: {error}")))?;
|
|
241
|
+
if journal.version != JOURNAL_VERSION {
|
|
242
|
+
return Err(MagGraphError::Index(format!(
|
|
243
|
+
"unsupported batch journal version {}",
|
|
244
|
+
journal.version
|
|
245
|
+
)));
|
|
246
|
+
}
|
|
247
|
+
for snapshot in journal.snapshots {
|
|
248
|
+
crate::security::validate_relative_node_path(&snapshot.relative_path)?;
|
|
249
|
+
crate::security::validate_relative_node_path(&snapshot.backup)?;
|
|
250
|
+
let contents = fs::read_to_string(journal_dir.join(&snapshot.backup)).map_err(|error| {
|
|
251
|
+
MagGraphError::Index(format!("failed to read batch backup: {error}"))
|
|
252
|
+
})?;
|
|
253
|
+
let (metadata, body) = crate::node::parse_markdown_node(&contents, &snapshot.relative_path)
|
|
254
|
+
.map_err(|message| MagGraphError::NodeParse {
|
|
255
|
+
path: snapshot.relative_path.clone(),
|
|
256
|
+
message,
|
|
257
|
+
})?;
|
|
258
|
+
Node {
|
|
259
|
+
metadata,
|
|
260
|
+
body,
|
|
261
|
+
relative_path: snapshot.relative_path,
|
|
262
|
+
}
|
|
263
|
+
.write_to(root)?;
|
|
264
|
+
}
|
|
265
|
+
fs::remove_dir_all(journal_dir).map_err(|error| {
|
|
266
|
+
MagGraphError::Index(format!("failed to clear recovered batch journal: {error}"))
|
|
267
|
+
})
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
fn sync_write(path: &Path, contents: &[u8]) -> Result<()> {
|
|
271
|
+
let mut file = File::create(path).map_err(|error| {
|
|
272
|
+
MagGraphError::Index(format!("failed to create {}: {error}", path.display()))
|
|
273
|
+
})?;
|
|
274
|
+
file.write_all(contents)
|
|
275
|
+
.and_then(|_| file.sync_all())
|
|
276
|
+
.map_err(|error| {
|
|
277
|
+
MagGraphError::Index(format!("failed to sync {}: {error}", path.display()))
|
|
278
|
+
})
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
#[cfg(test)]
|
|
282
|
+
mod tests {
|
|
283
|
+
use std::fs;
|
|
284
|
+
|
|
285
|
+
use tempfile::TempDir;
|
|
286
|
+
|
|
287
|
+
use super::*;
|
|
288
|
+
|
|
289
|
+
fn index() -> (TempDir, GraphIndex) {
|
|
290
|
+
let temp = TempDir::new().expect("tempdir");
|
|
291
|
+
for id in ["a", "b"] {
|
|
292
|
+
fs::write(
|
|
293
|
+
temp.path().join(format!("{id}.md")),
|
|
294
|
+
format!("---\nid: {id}\ntype: decision\n---\nBody {id}.\n"),
|
|
295
|
+
)
|
|
296
|
+
.expect("write node");
|
|
297
|
+
}
|
|
298
|
+
let index = GraphIndex::open(temp.path()).expect("open");
|
|
299
|
+
(temp, index)
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
#[test]
|
|
303
|
+
fn batch_applies_reviewed_operations() {
|
|
304
|
+
let (_temp, mut index) = index();
|
|
305
|
+
let result = apply_memory_batch(
|
|
306
|
+
&mut index,
|
|
307
|
+
&[
|
|
308
|
+
MemoryBatchOperation::UpdateBody {
|
|
309
|
+
id: "a".to_string(),
|
|
310
|
+
body: "Updated.".to_string(),
|
|
311
|
+
},
|
|
312
|
+
MemoryBatchOperation::Suppress {
|
|
313
|
+
id: "b".to_string(),
|
|
314
|
+
reason: Some("stale".to_string()),
|
|
315
|
+
},
|
|
316
|
+
],
|
|
317
|
+
)
|
|
318
|
+
.expect("batch");
|
|
319
|
+
|
|
320
|
+
assert_eq!(result.applied, ["update:a", "suppress:b"]);
|
|
321
|
+
assert_eq!(index.read_node("a").unwrap().body, "Updated.\n");
|
|
322
|
+
assert!(crate::query::is_suppressed(&index.read_node("b").unwrap()));
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
#[test]
|
|
326
|
+
fn batch_prevalidation_prevents_partial_changes() {
|
|
327
|
+
let (_temp, mut index) = index();
|
|
328
|
+
let error = apply_memory_batch(
|
|
329
|
+
&mut index,
|
|
330
|
+
&[
|
|
331
|
+
MemoryBatchOperation::UpdateBody {
|
|
332
|
+
id: "a".to_string(),
|
|
333
|
+
body: "Must not persist.".to_string(),
|
|
334
|
+
},
|
|
335
|
+
MemoryBatchOperation::Suppress {
|
|
336
|
+
id: "missing".to_string(),
|
|
337
|
+
reason: None,
|
|
338
|
+
},
|
|
339
|
+
],
|
|
340
|
+
)
|
|
341
|
+
.unwrap_err();
|
|
342
|
+
|
|
343
|
+
assert!(matches!(error, MagGraphError::NodeNotFound { .. }));
|
|
344
|
+
assert_eq!(index.read_node("a").unwrap().body, "Body a.\n");
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
#[test]
|
|
348
|
+
fn reopen_recovers_a_partially_applied_batch() {
|
|
349
|
+
let (temp, index) = index();
|
|
350
|
+
let snapshots = snapshots(
|
|
351
|
+
&index,
|
|
352
|
+
&[MemoryBatchOperation::Merge {
|
|
353
|
+
target_id: "a".to_string(),
|
|
354
|
+
source_id: "b".to_string(),
|
|
355
|
+
}],
|
|
356
|
+
)
|
|
357
|
+
.expect("snapshots");
|
|
358
|
+
let _journal = prepare_journal(temp.path(), &snapshots).expect("journal");
|
|
359
|
+
|
|
360
|
+
fs::write(
|
|
361
|
+
temp.path().join("a.md"),
|
|
362
|
+
"---\nid: a\ntype: decision\n---\nPartially updated.\n",
|
|
363
|
+
)
|
|
364
|
+
.expect("partial target");
|
|
365
|
+
fs::remove_file(temp.path().join("b.md")).expect("partial delete");
|
|
366
|
+
|
|
367
|
+
let reopened = GraphIndex::open(temp.path()).expect("recover and reopen");
|
|
368
|
+
assert_eq!(reopened.read_node("a").unwrap().body, "Body a.\n");
|
|
369
|
+
assert_eq!(reopened.read_node("b").unwrap().body, "Body b.\n");
|
|
370
|
+
assert!(
|
|
371
|
+
!transaction_root(temp.path()).exists()
|
|
372
|
+
|| transaction_root(temp.path())
|
|
373
|
+
.read_dir()
|
|
374
|
+
.unwrap()
|
|
375
|
+
.next()
|
|
376
|
+
.is_none()
|
|
377
|
+
);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
@@ -8,7 +8,9 @@ use walkdir::WalkDir;
|
|
|
8
8
|
use crate::error::{MagGraphError, Result};
|
|
9
9
|
use crate::graph::GraphAdjacency;
|
|
10
10
|
use crate::node::{NewNode, Node, NodeMetadata};
|
|
11
|
-
use crate::query::{
|
|
11
|
+
use crate::query::{
|
|
12
|
+
changed_since, GraphChange, HybridQueryOptions, HybridSearchResult, QueryOptions, SearchResult,
|
|
13
|
+
};
|
|
12
14
|
use crate::recall::{recall_bundle, RecallBundle};
|
|
13
15
|
use crate::security::validate_relative_node_path;
|
|
14
16
|
use crate::sync::WritePolicy;
|
|
@@ -63,6 +65,8 @@ impl GraphIndex {
|
|
|
63
65
|
)));
|
|
64
66
|
}
|
|
65
67
|
|
|
68
|
+
crate::batch::recover_incomplete_batches(&root_path)?;
|
|
69
|
+
|
|
66
70
|
let mut index = Self {
|
|
67
71
|
root_path,
|
|
68
72
|
by_id: HashMap::new(),
|
|
@@ -202,6 +206,11 @@ impl GraphIndex {
|
|
|
202
206
|
crate::query::search_index(self, options)
|
|
203
207
|
}
|
|
204
208
|
|
|
209
|
+
/// Explainable hybrid retrieval over lexical, graph, recency, and semantic signals.
|
|
210
|
+
pub fn hybrid_search(&self, options: &HybridQueryOptions) -> Result<Vec<HybridSearchResult>> {
|
|
211
|
+
crate::query::hybrid_search_index(self, options)
|
|
212
|
+
}
|
|
213
|
+
|
|
205
214
|
/// Node ids that link to `id`.
|
|
206
215
|
pub fn backlinks(&self, id: &str) -> Result<Vec<String>> {
|
|
207
216
|
if !self.contains(id) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
pub mod agent;
|
|
2
|
+
pub mod batch;
|
|
2
3
|
pub mod config;
|
|
3
4
|
pub mod error;
|
|
4
5
|
pub mod graph;
|
|
@@ -20,6 +21,9 @@ pub use agent::{
|
|
|
20
21
|
render_skill_md, write_mcp_scaffold, write_skill_md, EdgePattern, EdgeSource, GraphSchema,
|
|
21
22
|
McpScaffoldContext, SchemaEdge, SkillRenderContext,
|
|
22
23
|
};
|
|
24
|
+
pub use batch::{
|
|
25
|
+
apply_memory_batch, validate_memory_batch, MemoryBatchOperation, MemoryBatchResult,
|
|
26
|
+
};
|
|
23
27
|
pub use config::{
|
|
24
28
|
LakehouseCacheConfig, LakehouseConfig, MagGraphConfig, RemoteSource, ResolvedConfig,
|
|
25
29
|
StorageConfig, StorageMode, SyncConfig, SyncRole, METADATA_DIR_NAME,
|
|
@@ -33,9 +37,15 @@ pub use lakehouse::{
|
|
|
33
37
|
ContentCache, ContentResolver, FileResolver, HttpResolver, LakehouseReader, NodeWithContent,
|
|
34
38
|
ResolvedContent, ResolverRegistry, S3StubResolver,
|
|
35
39
|
};
|
|
36
|
-
pub use memory::{
|
|
40
|
+
pub use memory::{
|
|
41
|
+
new_memory_node, new_memory_node_with_context, validate_memory_type, MemoryContext,
|
|
42
|
+
MemoryNodeKind, MEMORY_TYPES,
|
|
43
|
+
};
|
|
37
44
|
pub use node::{NewNode, Node, NodeMetadata};
|
|
38
|
-
pub use query::{
|
|
45
|
+
pub use query::{
|
|
46
|
+
changed_since, hybrid_search_index, search_index, GraphChange, HybridQueryOptions,
|
|
47
|
+
HybridSearchResult, HybridWeights, QueryOptions, SearchResult,
|
|
48
|
+
};
|
|
39
49
|
pub use recall::{recall_bundle, RecallBundle};
|
|
40
50
|
pub use security::{assert_path_within_root, validate_http_uri_host, validate_relative_node_path};
|
|
41
51
|
pub use sync::{
|