cortexm 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. context_m.py +17 -0
  2. cortexm/__init__.py +45 -0
  3. cortexm/accel.py +403 -0
  4. cortexm/api/__init__.py +0 -0
  5. cortexm/api/chaos.py +118 -0
  6. cortexm/api/memory.py +635 -0
  7. cortexm/bench/__init__.py +0 -0
  8. cortexm/bench/abilities.py +311 -0
  9. cortexm/bench/baselines.py +89 -0
  10. cortexm/bench/beam_loader.py +317 -0
  11. cortexm/bench/generator.py +376 -0
  12. cortexm/bench/harness.py +211 -0
  13. cortexm/bench/messy.py +218 -0
  14. cortexm/bench/micro.py +251 -0
  15. cortexm/bench/ood.py +443 -0
  16. cortexm/bench/run.py +137 -0
  17. cortexm/bridge/__init__.py +0 -0
  18. cortexm/bridge/dates.py +178 -0
  19. cortexm/bridge/decoders.py +204 -0
  20. cortexm/bridge/enrich.py +255 -0
  21. cortexm/bridge/extractor.py +316 -0
  22. cortexm/bridge/fallback.py +332 -0
  23. cortexm/bridge/onnx_runtime.py +158 -0
  24. cortexm/bridge/patterns.py +760 -0
  25. cortexm/bridge/ppr.py +104 -0
  26. cortexm/bridge/prefilter.py +188 -0
  27. cortexm/bridge/query_extract.py +420 -0
  28. cortexm/bridge/reader.py +1174 -0
  29. cortexm/bridge/rerank.py +204 -0
  30. cortexm/bridge/writer.py +492 -0
  31. cortexm/cli.py +295 -0
  32. cortexm/cognition/__init__.py +53 -0
  33. cortexm/cognition/abstraction.py +192 -0
  34. cortexm/cognition/analogy.py +159 -0
  35. cortexm/cognition/engine.py +204 -0
  36. cortexm/cognition/gaps.py +365 -0
  37. cortexm/cognition/scanner.py +204 -0
  38. cortexm/config.py +375 -0
  39. cortexm/cortexm.py +8 -0
  40. cortexm/enterprise/__init__.py +0 -0
  41. cortexm/enterprise/audit.py +178 -0
  42. cortexm/enterprise/governance.py +239 -0
  43. cortexm/errors.py +35 -0
  44. cortexm/features/__init__.py +0 -0
  45. cortexm/features/git.py +204 -0
  46. cortexm/features/prefetch.py +88 -0
  47. cortexm/features/zk.py +105 -0
  48. cortexm/federation/__init__.py +39 -0
  49. cortexm/federation/crdt.py +275 -0
  50. cortexm/federation/fabric.py +109 -0
  51. cortexm/federation/hlc.py +80 -0
  52. cortexm/federation/node.py +145 -0
  53. cortexm/federation/schema_report.py +73 -0
  54. cortexm/federation/transport.py +164 -0
  55. cortexm/index/__init__.py +19 -0
  56. cortexm/index/nsg.py +386 -0
  57. cortexm/mcp/__init__.py +0 -0
  58. cortexm/mcp/server.py +985 -0
  59. cortexm/metrics.py +62 -0
  60. cortexm/migrate/__init__.py +0 -0
  61. cortexm/migrate/importers.py +192 -0
  62. cortexm/provenance/__init__.py +78 -0
  63. cortexm/provenance/agent.py +214 -0
  64. cortexm/provenance/cose.py +201 -0
  65. cortexm/provenance/scitt.py +258 -0
  66. cortexm/provenance/vc.py +250 -0
  67. cortexm/security/__init__.py +0 -0
  68. cortexm/security/crypto.py +162 -0
  69. cortexm/security/hashes.py +140 -0
  70. cortexm/security/injection.py +149 -0
  71. cortexm/security/mind.py +154 -0
  72. cortexm/security/pii.py +265 -0
  73. cortexm/security/rbac.py +169 -0
  74. cortexm/security/sandbox.py +131 -0
  75. cortexm/security/zk_hamming.py +142 -0
  76. cortexm/security/zk_sql.py +485 -0
  77. cortexm/server/__init__.py +0 -0
  78. cortexm/server/metrics.py +88 -0
  79. cortexm/server/rest.py +936 -0
  80. cortexm/server/sparql.py +984 -0
  81. cortexm/text/__init__.py +0 -0
  82. cortexm/text/dissim.py +252 -0
  83. cortexm/text/embedder.py +155 -0
  84. cortexm/text/fuzzy.py +218 -0
  85. cortexm/text/idiolect.py +253 -0
  86. cortexm/text/labse.py +374 -0
  87. cortexm/text/tokenizer.py +79 -0
  88. cortexm/trace/__init__.py +0 -0
  89. cortexm/trace/blob_arena.py +277 -0
  90. cortexm/trace/consolidate.py +337 -0
  91. cortexm/trace/contradictions.py +69 -0
  92. cortexm/trace/dedup.py +114 -0
  93. cortexm/trace/edges.py +214 -0
  94. cortexm/trace/fact.py +121 -0
  95. cortexm/trace/fade.py +245 -0
  96. cortexm/trace/lifecycle.py +112 -0
  97. cortexm/trace/rebuild.py +173 -0
  98. cortexm/trace/rules.py +171 -0
  99. cortexm/trace/store.py +680 -0
  100. cortexm/trace/structural.py +183 -0
  101. cortexm/trace/tmt.py +335 -0
  102. cortexm/util.py +148 -0
  103. cortexm/vsa/__init__.py +0 -0
  104. cortexm/vsa/attribution.py +149 -0
  105. cortexm/vsa/cleanup.py +161 -0
  106. cortexm/vsa/codecs.py +397 -0
  107. cortexm/vsa/hologram_overlay.py +139 -0
  108. cortexm/vsa/index.py +163 -0
  109. cortexm/vsa/ops.py +149 -0
  110. cortexm/vsa/palace.py +446 -0
  111. cortexm/vsa/role_vectors.py +236 -0
  112. cortexm/vsa/slb.py +78 -0
  113. cortexm/vsa/tlsh_trie.py +137 -0
  114. cortexm/vsa/working_memory.py +249 -0
  115. cortexm-0.3.0.dist-info/METADATA +482 -0
  116. cortexm-0.3.0.dist-info/RECORD +120 -0
  117. cortexm-0.3.0.dist-info/WHEEL +5 -0
  118. cortexm-0.3.0.dist-info/entry_points.txt +2 -0
  119. cortexm-0.3.0.dist-info/licenses/LICENSE +190 -0
  120. cortexm-0.3.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,178 @@
1
+ """Deterministic date-expression resolution (μ=0 temporal parsing).
2
+
3
+ Extracts absolute and relative date expressions from text, resolved
4
+ against the conversation timestamp — the substrate for bi-temporal
5
+ valid times and BEAM temporal-reasoning abilities.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ from datetime import datetime, timedelta
12
+
13
+ from cortexm.util import month_name, month_number
14
+
15
+ MONTH_RE = r"(?:January|February|March|April|May|June|July|August|September|October|November|December)"
16
+
17
+ ABS_PATTERNS = [
18
+ # March 3, 2024 / March 3rd 2024 / March 3, 2024
19
+ (re.compile(rf"\b(?:on\s+)?({MONTH_RE})\.?\s+(\d{{1,2}})(?:st|nd|rd|th)?,?\s+(\d{{4}})\b", re.I), "day"),
20
+ # 3 March 2024
21
+ (re.compile(rf"\b(\d{{1,2}})(?:st|nd|rd|th)?\s+(?:of\s+)?({MONTH_RE})\.?,?\s+(\d{{4}})\b", re.I), "day_r"),
22
+ # March 3 (no year — assume year of context ts, prefer past)
23
+ (re.compile(rf"\b(?:on\s+)?({MONTH_RE})\.?\s+(\d{{1,2}})(?:st|nd|rd|th)?\b(?!\s*,?\s*\d{{4}})", re.I), "day_noyear"),
24
+ # March 2024 (month granularity)
25
+ (re.compile(rf"\bin\s+({MONTH_RE})\.?,?\s+(\d{{4}})\b", re.I), "month"),
26
+ (re.compile(rf"\b({MONTH_RE})\s+(\d{{4}})\b"), "month"),
27
+ # 2025-08-15 (ISO day)
28
+ (re.compile(r"\b(\d{4})-(\d{2})-(\d{2})\b"), "iso_day"),
29
+ # 2025-08 / 2025/08 / "2025 08" (numeric year-month; before bare year)
30
+ (re.compile(r"\b(\d{4})[-/ ](\d{1,2})\b"), "ym_num"),
31
+ (re.compile(r"\bin\s+(\d{4})\b"), "year"),
32
+ ]
33
+
34
+ REL_PATTERNS = [
35
+ (re.compile(r"\byesterday\b", re.I), "yesterday"),
36
+ (re.compile(r"\b(today|this morning|right now)\b", re.I), "today"),
37
+ (re.compile(r"\btomorrow\b", re.I), "tomorrow"),
38
+ (re.compile(r"\blast\s+night\b", re.I), "yesterday"),
39
+ (re.compile(rf"\blast\s+({MONTH_RE})\b", re.I), "last_month"),
40
+ (re.compile(r"\blast\s+week\b", re.I), "last_week"),
41
+ (re.compile(r"\blast\s+month\b", re.I), "last_month_rel"),
42
+ (re.compile(r"\blast\s+year\b", re.I), "last_year"),
43
+ (re.compile(r"\b(\d+|a|an|one|two|three|four|five|six|few|several|couple)\s+(day|week|month|year)s?\s+ago\b", re.I), "ago"),
44
+ (re.compile(rf"\b(?:in|since)\s+({MONTH_RE})\b(?!\s*,?\s*\d{{4}})", re.I), "this_year_month"),
45
+ (re.compile(r"\bsince\s+(\d{4})\b"), "year"),
46
+ ]
47
+
48
+ WEEKDAYS = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]
49
+ LAST_WEEKDAY = re.compile(rf"\blast\s+({'|'.join(WEEKDAYS)})\b", re.I)
50
+
51
+
52
+ def _month_num(name: str) -> int:
53
+ return month_number(name)
54
+
55
+
56
+ def _mk(year: int, month: int, day: int) -> str:
57
+ return f"{year:04d}-{month:02d}-{day:02d}"
58
+
59
+
60
+ def resolve_relative(label: str, m: re.Match, ts: datetime) -> tuple[str, str] | None:
61
+ """Return (iso_date, surface) for a relative expression."""
62
+ g = m.group(0).lower()
63
+ if label == "yesterday":
64
+ d = ts - timedelta(days=1)
65
+ return _mk(d.year, d.month, d.day), g
66
+ if label == "today":
67
+ return _mk(ts.year, ts.month, ts.day), g
68
+ if label == "tomorrow":
69
+ d = ts + timedelta(days=1)
70
+ return _mk(d.year, d.month, d.day), g
71
+ if label == "last_week":
72
+ d = ts - timedelta(days=7)
73
+ return _mk(d.year, d.month, d.day), g
74
+ if label == "last_month_rel":
75
+ d = ts.replace(day=1) - timedelta(days=1)
76
+ return _mk(d.year, d.month, 1), g
77
+ if label == "last_year":
78
+ return _mk(ts.year - 1, 1, 1), g
79
+ if label == "last_month":
80
+ mm = _month_num(m.group(1))
81
+ y = ts.year if mm <= ts.month else ts.year - 1
82
+ return _mk(y, mm, 1), g
83
+ if label == "ago":
84
+ raw = m.group(1).lower()
85
+ words = {"a": 1, "an": 1, "one": 1, "two": 2, "three": 3,
86
+ "four": 4, "five": 5, "six": 6, "few": 3, "several": 4,
87
+ "couple": 2}
88
+ n = words.get(raw, 1) if not raw.isdigit() else int(raw)
89
+ unit = m.group(2).lower()
90
+ if unit.startswith("day"):
91
+ d = ts - timedelta(days=n)
92
+ elif unit.startswith("week"):
93
+ d = ts - timedelta(weeks=n)
94
+ elif unit.startswith("month"):
95
+ d = ts.replace(day=1)
96
+ for _ in range(n):
97
+ d = (d.replace(day=1) - timedelta(days=1)).replace(day=1)
98
+ return _mk(d.year, d.month, d.day), g
99
+ else:
100
+ return _mk(ts.year - n, ts.month, ts.day), g
101
+ return _mk(d.year, d.month, d.day), g
102
+ if label == "this_year_month":
103
+ mm = _month_num(m.group(1))
104
+ y = ts.year if mm <= ts.month else ts.year - 1
105
+ return _mk(y, mm, 1), g
106
+ if label == "year":
107
+ return f"{m.group(1)}-01-01", g
108
+ return None
109
+
110
+
111
+ def find_dates(text: str, ts: datetime) -> list[dict]:
112
+ """All date expressions: [{span, iso, surface, granularity}]."""
113
+ out: list[dict] = []
114
+ taken: list[tuple[int, int]] = []
115
+
116
+ def overlaps(s, e):
117
+ return any(not (e <= a or s >= b) for a, b in taken)
118
+
119
+ for rx, label in ABS_PATTERNS:
120
+ for m in rx.finditer(text):
121
+ if overlaps(m.start(), m.end()):
122
+ continue
123
+ try:
124
+ if label == "day":
125
+ mo, d, y = _month_num(m.group(1)), int(m.group(2)), int(m.group(3))
126
+ elif label == "day_r":
127
+ d, mo, y = int(m.group(1)), _month_num(m.group(2)), int(m.group(3))
128
+ elif label == "day_noyear":
129
+ mo, d = _month_num(m.group(1)), int(m.group(2))
130
+ y = ts.year if (mo, d) <= (ts.month, ts.day) else ts.year - 1
131
+ elif label == "year":
132
+ y = int(m.group(1))
133
+ mo, d = 1, 1
134
+ elif label == "iso_day":
135
+ y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
136
+ elif label == "ym_num":
137
+ y, mo = int(m.group(1)), int(m.group(2))
138
+ if not 1 <= mo <= 12: # not a month — e.g. "2025 30"
139
+ continue
140
+ d = 1
141
+ else: # month
142
+ mo, y = _month_num(m.group(1)), int(m.group(2))
143
+ d = 1
144
+ iso = _mk(y, mo, d)
145
+ except (ValueError, TypeError):
146
+ continue
147
+ out.append({"span": (m.start(), m.end()), "iso": iso,
148
+ "surface": m.group(0), "granularity": label})
149
+ taken.append((m.start(), m.end()))
150
+
151
+ for rx, label in REL_PATTERNS:
152
+ for m in rx.finditer(text):
153
+ if overlaps(m.start(), m.end()):
154
+ continue
155
+ r = resolve_relative(label, m, ts)
156
+ if r:
157
+ iso, surface = r
158
+ out.append({"span": (m.start(), m.end()), "iso": iso,
159
+ "surface": m.group(0), "granularity": label})
160
+ taken.append((m.start(), m.end()))
161
+
162
+ for m in LAST_WEEKDAY.finditer(text):
163
+ if overlaps(m.start(), m.end()):
164
+ continue
165
+ wd = WEEKDAYS.index(m.group(1).lower())
166
+ delta = (ts.weekday() - wd) % 7 or 7
167
+ d = ts - timedelta(days=delta)
168
+ out.append({"span": (m.start(), m.end()), "iso": _mk(d.year, d.month, d.day),
169
+ "surface": m.group(0), "granularity": "weekday"})
170
+ taken.append((m.start(), m.end()))
171
+
172
+ out.sort(key=lambda d: d["span"][0])
173
+ return out
174
+
175
+
176
+ def first_date(text: str, ts: datetime) -> str | None:
177
+ ds = find_dates(text, ts)
178
+ return ds[0]["iso"] if ds else None
@@ -0,0 +1,204 @@
1
+ """Swappable retrieval decoders — NSR-inspired (ESWEEK24).
2
+
3
+ arXiv insight: the VSA core is task-agnostic; only the decoder
4
+ changes. Context-M's reader was hardcoded to format facts for an LLM
5
+ prompt (the `_context_block` method). This module extracts that
6
+ formatter into a pluggable decoder interface so the SAME palace + Trace
7
+ can serve multiple output formats:
8
+
9
+ - LLMPromptDecoder — current behavior, formats facts as a
10
+ "[Memory — Known facts]" block for LLM
11
+ context-stuffing
12
+ - RDFDecoder — export facts as RDF/N3 triples for external
13
+ graph DBs (SPARQL queryable)
14
+ - DatalogDecoder — emit facts as Datalog clauses for the
15
+ contradiction engine to consume
16
+ - JSONDecoder — return facts as plain JSON for API responses
17
+
18
+ All decoders take the same inputs (query, facts, scores, notes,
19
+ store) and return a string. The reader picks one via
20
+ `reader.with_decoder(decoder)` or by passing `decoder=` to .search().
21
+ The default remains LLMPromptDecoder so existing callers don't break.
22
+
23
+ Why this matters: the palace / Trace substrate is now reusable for
24
+ non-LLM workloads (RDF export, contradiction proofs, audit JSON)
25
+ without duplicating the retrieval pipeline.
26
+ """
27
+ from __future__ import annotations
28
+
29
+ import json
30
+ from typing import Iterable, Protocol
31
+
32
+
33
+ class Decoder(Protocol):
34
+ """Pluggable retrieval-output decoder.
35
+
36
+ A decoder takes the post-fusion fact list + scores + procedural
37
+ notes and renders them in a domain-specific format. The reader
38
+ pipeline (intent plan → palace search → fusion → notes) is
39
+ identical across decoders — only the output shape changes.
40
+ """
41
+ name: str
42
+
43
+ def render(self, *, query: str, intent: str, facts: list,
44
+ scores: dict, notes: list[str] | None,
45
+ store=None) -> str: ...
46
+
47
+
48
+ # ---------- LLM prompt decoder (current behavior) ----------------------
49
+
50
+ class LLMPromptDecoder:
51
+ """Format facts as a context block for LLM context-stuffing.
52
+
53
+ This is the original reader._context_block output. Kept verbatim
54
+ so existing prompts continue to work.
55
+ """
56
+ name = "llm_prompt"
57
+
58
+ def render(self, *, query, intent, facts, scores, notes, store=None) -> str:
59
+ lines = ["[Memory — Known facts]"]
60
+ if not facts and not notes:
61
+ lines.append("(no verified facts found for this query)")
62
+ for f in facts:
63
+ chunk = store.get_chunk(f.source_id) if (
64
+ store and f.source_id) else None
65
+ snippet = ""
66
+ if chunk:
67
+ snippet = chunk["text"].replace("\n", " ")[:80]
68
+ if len(chunk["text"]) > 80:
69
+ snippet += "..."
70
+ lines.append(
71
+ f"- {f.display()} [valid {f.valid_window()}; "
72
+ f"learned {f.tx_from[:10]}; conf {f.confidence:.2f}; "
73
+ f"id {f.id[:8]}; src #{f.source_hash[:8]}; \"{snippet}\"]")
74
+ for n in (notes or []):
75
+ lines.append(f"- {n}")
76
+ return "\n".join(lines)
77
+
78
+
79
+ # ---------- RDF / N3 decoder ------------------------------------------
80
+
81
+ class RDFDecoder:
82
+ """Export facts as RDF/N3 triples.
83
+
84
+ Format: <subject> <relation> <value> .
85
+ Subjects/values are URI-escaped; string literals quoted. Useful
86
+ for piping Context-M's verified facts into an external graph DB
87
+ (e.g. Apache Jena, BlazeGraph) for SPARQL queries.
88
+
89
+ NOTE: namespaces are bare (no @prefix). Callers can post-process
90
+ to add prefix declarations if desired.
91
+ """
92
+ name = "rdf"
93
+
94
+ def render(self, *, query, intent, facts, scores, notes, store=None) -> str:
95
+ out: list[str] = []
96
+ for f in facts:
97
+ out.append(f"{self._term(f.subject)} "
98
+ f"{self._term(f.relation)} "
99
+ f"{self._term(f.value)} .")
100
+ if notes:
101
+ for n in notes:
102
+ out.append(f"# note: {n}")
103
+ return "\n".join(out)
104
+
105
+ @staticmethod
106
+ def _term(s: str) -> str:
107
+ # bare URI if alnum+CWD, else quoted literal
108
+ if not s:
109
+ return '""'
110
+ # treat CamelCase / dotted names as URIs in the local : namespace
111
+ if all(c.isalnum() or c in "._-:/" for c in s) and not s[0].isdigit():
112
+ return f":{s}"
113
+ # quoted literal with N3 escapes
114
+ return '"' + s.replace('"', '\\"') + '"'
115
+
116
+
117
+ # ---------- Datalog decoder -------------------------------------------
118
+
119
+ class DatalogDecoder:
120
+ """Emit facts as Datalog clauses for the contradiction engine.
121
+
122
+ Format: relation(subject, value).
123
+ Used by the contradiction / lifecycle engines to reason over
124
+ verified facts as logic predicates. The current Trace.rules
125
+ engine already speaks this dialect — this decoder exposes the
126
+ same format for external callers.
127
+ """
128
+ name = "datalog"
129
+
130
+ def render(self, *, query, intent, facts, scores, notes, store=None) -> str:
131
+ out: list[str] = []
132
+ for f in facts:
133
+ out.append(f"{f.relation}({self._atom(f.subject)}, "
134
+ f"{self._atom(f.value)}).")
135
+ if notes:
136
+ for n in notes:
137
+ out.append(f"% note: {n}")
138
+ return "\n".join(out)
139
+
140
+ @staticmethod
141
+ def _atom(s: str) -> str:
142
+ # Datalog atom — lowercase, spaces → underscores
143
+ if not s:
144
+ return "_"
145
+ a = s.lower().replace(" ", "_").replace("-", "_")
146
+ # quote if starts with digit or contains non-atom chars
147
+ if a[0].isdigit() or not all(c.isalnum() or c == "_" for c in a):
148
+ return f'"{s}"'
149
+ return a
150
+
151
+
152
+ # ---------- JSON decoder ----------------------------------------------
153
+
154
+ class JSONDecoder:
155
+ """Return facts as a JSON array of triples.
156
+
157
+ Each entry: {subject, relation, value, valid_from, valid_to,
158
+ confidence, id, score}. Used by REST API responses and by
159
+ audit / federation tools that consume structured facts.
160
+ """
161
+ name = "json"
162
+
163
+ def render(self, *, query, intent, facts, scores, notes, store=None) -> str:
164
+ out: list[dict] = []
165
+ for f in facts:
166
+ out.append({
167
+ "subject": f.subject,
168
+ "relation": f.relation,
169
+ "value": f.value,
170
+ "valid_from": f.valid_from,
171
+ "valid_to": f.valid_to,
172
+ "tx_from": f.tx_from,
173
+ "confidence": round(f.confidence, 4),
174
+ "id": f.id,
175
+ "score": round(scores.get(f.id, 0.0), 4) if scores else None,
176
+ })
177
+ if notes:
178
+ out.append({"_notes": notes})
179
+ return json.dumps(out, default=str, indent=2)
180
+
181
+
182
+ # ---------- Registry --------------------------------------------------
183
+
184
+ DECODERS: dict[str, type] = {
185
+ "llm_prompt": LLMPromptDecoder,
186
+ "rdf": RDFDecoder,
187
+ "datalog": DatalogDecoder,
188
+ "json": JSONDecoder,
189
+ }
190
+
191
+
192
+ def get_decoder(name: str = "llm_prompt") -> "Decoder":
193
+ """Look up a decoder by name; raises ValueError for unknown names."""
194
+ cls = DECODERS.get(name)
195
+ if cls is None:
196
+ raise ValueError(
197
+ f"unknown decoder '{name}' — known: {list(DECODERS)}")
198
+ return cls()
199
+
200
+
201
+ __all__ = [
202
+ "Decoder", "LLMPromptDecoder", "RDFDecoder", "DatalogDecoder",
203
+ "JSONDecoder", "DECODERS", "get_decoder",
204
+ ]
@@ -0,0 +1,255 @@
1
+ """Async LLM enrichment fallback — graceful degradation off the μ=0 path.
2
+
3
+ The μ=0 protocol guarantees the SYNCHRONOUS ingest path never calls an LLM.
4
+ That determinism is the product's cost/speed moat, but 60 hand-written
5
+ patterns will inevitably miss messy, indirect, or non-English phrasing.
6
+ This module is the plan's acknowledged fallback: AFTER a chunk is stored
7
+ (and its BLAKE3 hash sealed), chunks whose text yielded ZERO pattern
8
+ candidates are queued for a second-pass LLM extraction. Enriched facts:
9
+
10
+ * carry provenance {"pattern": "llm_enrichment", "extractor_model": ...}
11
+ so every enriched fact is auditable and distinguishable from μ=0 facts;
12
+ * are confidence-capped (default 0.85) so deterministic facts always win
13
+ conflicts against enriched ones;
14
+ * still pass the full InjecMEM / MINJA quarantine + lifecycle pipeline;
15
+ * bump metrics.llm_calls so the μ=0 audit trail stays honest — an auditor
16
+ sees exactly which phase spent LLM budget.
17
+
18
+ Usage (explicit, never automatic):
19
+ report = memory.enrich(user_id="alice") # sync
20
+ report = memory.enrich_async(user_id="alice") # background thread
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import json
26
+ import os
27
+ import subprocess
28
+ import tempfile
29
+ import threading
30
+ from dataclasses import dataclass
31
+
32
+ from cortexm import metrics
33
+ from cortexm.bridge.patterns import Candidate
34
+ from cortexm.bridge.writer import MemoryWriter
35
+ from cortexm.util import normalize
36
+
37
+ _HERE = os.path.dirname(os.path.abspath(__file__))
38
+ _REPO_ROOT = os.path.dirname(os.path.dirname(_HERE)) # .../context-m
39
+ DEFAULT_EXTRACTOR_JS = os.path.join(_REPO_ROOT, "benchmarks", "llm",
40
+ "extract_facts.mjs")
41
+ DEFAULT_NODE = os.environ.get("CORTEXM_NODE_BIN", "node")
42
+ SDK_FALLBACKS = [
43
+ "/home/z/.bun/install/global/node_modules/z-ai-web-dev-sdk/dist/index.js",
44
+ ]
45
+
46
+ ENRICH_CONFIDENCE_CAP = 0.85
47
+
48
+
49
+ @dataclass
50
+ class EnrichmentReport:
51
+ chunks_total: int = 0
52
+ chunks_eligible: int = 0 # zero pattern candidates
53
+ llm_calls: int = 0
54
+ llm_tokens: int = 0
55
+ facts_extracted: int = 0
56
+ facts_committed: int = 0
57
+ quarantined: int = 0
58
+ seconds: float = 0.0
59
+ extractor_model: str | None = None
60
+
61
+ def to_dict(self) -> dict:
62
+ return dict(self.__dict__)
63
+
64
+
65
+ class NodeLLMExtractor:
66
+ """Subprocess bridge to benchmarks/llm/extract_facts.mjs (z-ai SDK).
67
+
68
+ Injectable and replaceable: tests pass a plain Python callable with the
69
+ same (texts, subjects) -> list[list[dict]] contract instead.
70
+ """
71
+
72
+ def __init__(self, script: str | None = None, concurrency: int = 4) -> None:
73
+ self.script = script or DEFAULT_EXTRACTOR_JS
74
+ self.concurrency = concurrency
75
+
76
+ def __call__(self, texts: list[str], subjects: list[str | None]) -> list[list[dict]]:
77
+ if not texts:
78
+ return []
79
+ items = [{"id": i, "text": t, "subject": s or None}
80
+ for i, (t, s) in enumerate(zip(texts, subjects))]
81
+ with tempfile.TemporaryDirectory(prefix="cortexm-enrich-") as td:
82
+ inp = os.path.join(td, "in.jsonl")
83
+ outp = os.path.join(td, "out.jsonl")
84
+ with open(inp, "w", encoding="utf-8") as fh:
85
+ fh.writelines(json.dumps(it) + "\n" for it in items)
86
+ env = dict(os.environ)
87
+ env.setdefault("LLM_CONCURRENCY", str(self.concurrency))
88
+ env.setdefault("LLM_CACHE_DIR",
89
+ os.path.join(_REPO_ROOT, "benchmarks", "llm", ".cache"))
90
+ proc = subprocess.run(
91
+ [DEFAULT_NODE, self.script, inp, outp, "--enrich"],
92
+ capture_output=True, text=True, timeout=1800, env=env)
93
+ if proc.returncode != 0 or not os.path.exists(outp):
94
+ raise RuntimeError(f"enrichment extractor failed: "
95
+ f"{proc.stderr[-500:]}")
96
+ rows = [json.loads(l) for l in open(outp, encoding="utf-8")
97
+ if l.strip()]
98
+ rows.sort(key=lambda r: int(r.get("id", 0)))
99
+ out: list[list[dict]] = []
100
+ for r in rows:
101
+ facts = [f for f in (r.get("facts") or [])
102
+ if isinstance(f, dict) and f.get("subject")
103
+ and f.get("relation") and f.get("value")]
104
+ out.append(facts)
105
+ # keep alignment with input length
106
+ while len(out) < len(texts):
107
+ out.append([])
108
+ return out
109
+
110
+
111
+ def _default_extractor():
112
+ return NodeLLMExtractor()
113
+
114
+
115
+ def find_eligible_chunks(writer: MemoryWriter, user_id: str | None,
116
+ limit: int | None = None) -> list[dict]:
117
+ """Chunks where the μ=0 extractor produced no REAL candidates.
118
+
119
+ Low-confidence ``mention_fallback`` mentions (pattern="mention_fallback",
120
+ conf 0.35) fire on almost any capitalized text — they are lexicon noise,
121
+ not signal. A chunk is enrichment-eligible only when no genuine pattern
122
+ matched, which is exactly the "pattern confidence dropped" condition the
123
+ strategic plan describes for the async LLM fallback.
124
+ """
125
+ from cortexm.bridge.patterns import ExtractionContext
126
+ from cortexm.bridge.extractor import Extractor
127
+
128
+ extractor = writer.extractor if hasattr(writer, "extractor") else Extractor(writer.cfg)
129
+ eligible: list[dict] = []
130
+ for chunk in writer.store.all_chunks(user_id):
131
+ ctx = ExtractionContext(
132
+ user_id=chunk.get("user_id") or "default",
133
+ agent_id=chunk.get("agent_id"), run_id=chunk.get("run_id"),
134
+ ts=_parse_ts(chunk.get("ts")),
135
+ speaker="assistant" if chunk.get("source") in ("assistant", "ai", "bot") else "user",
136
+ subject_name=writer._name_of(chunk.get("user_id") or "default"),
137
+ lexicon=writer._lexicon(chunk.get("user_id") or "default"))
138
+ try:
139
+ cands = extractor.extract(chunk["text"], ctx)
140
+ real = [c for c in cands if c.pattern != "mention_fallback"]
141
+ except Exception:
142
+ real = []
143
+ if not real and len(chunk["text"].strip()) >= 20:
144
+ eligible.append(chunk)
145
+ if limit and len(eligible) >= limit:
146
+ break
147
+ return eligible
148
+
149
+
150
+ def _parse_ts(v):
151
+ if not v:
152
+ return None
153
+ from datetime import datetime, timezone
154
+ try:
155
+ return datetime.fromisoformat(v.replace("Z", "+00:00")) \
156
+ if isinstance(v, str) else v
157
+ except ValueError:
158
+ return None
159
+
160
+
161
+ def enrich(writer: MemoryWriter, user_id: str | None = None, *,
162
+ extractor=None, limit: int | None = None,
163
+ min_confidence: float | None = None,
164
+ dry_run: bool = False) -> EnrichmentReport:
165
+ """Second-pass LLM extraction over zero-signal chunks. Explicit call."""
166
+ import time
167
+
168
+ t0 = time.time()
169
+ rep = EnrichmentReport()
170
+ eligible = find_eligible_chunks(writer, user_id, limit)
171
+ rep.chunks_total = len(writer.store.all_chunks(user_id))
172
+ rep.chunks_eligible = len(eligible)
173
+ if dry_run or not eligible:
174
+ rep.seconds = round(time.time() - t0, 3)
175
+ return rep
176
+
177
+ extractor = extractor or _default_extractor()
178
+ texts = [c["text"] for c in eligible]
179
+ subjects = [writer._name_of(c.get("user_id") or "default") for c in eligible]
180
+ batch = extractor(texts, subjects)
181
+ rep.llm_calls += 1
182
+ metrics.bump_llm_call()
183
+
184
+ cap = ENRICH_CONFIDENCE_CAP
185
+ floor = min_confidence if min_confidence is not None \
186
+ else writer.cfg.min_confidence
187
+
188
+ for chunk, facts in zip(eligible, batch):
189
+ model = None
190
+ uid = chunk.get("user_id") or "default"
191
+ # name learning from enriched facts: if the LLM spotted "my name
192
+ # is X" in text the patterns missed, later facts in this batch can
193
+ # be re-subjected to X so they align with query entities.
194
+ learned_name = None
195
+ for f in facts:
196
+ if str(f.get("relation", "")).lower() == "name" and f.get("value"):
197
+ learned_name = str(f["value"]).strip()
198
+ if writer._name_of(uid) is None and learned_name:
199
+ writer._set_name(uid, learned_name)
200
+ break
201
+ subj_hint = learned_name or writer._name_of(uid) or uid
202
+ cands: list[Candidate] = []
203
+ for f in facts:
204
+ try:
205
+ conf = float(f.get("confidence", 0.7))
206
+ except (TypeError, ValueError):
207
+ conf = 0.7
208
+ conf = min(conf, cap)
209
+ if conf < floor:
210
+ continue
211
+ model = f.get("_model") or model
212
+ subj = str(f.get("subject") or "").strip()
213
+ # re-subject generic speaker labels to the learned persona name
214
+ # so enriched facts are findable by entity queries
215
+ if (not subj or subj.lower() in ("user", "the user", "speaker",
216
+ "i", "me")):
217
+ subj = subj_hint
218
+ cands.append(Candidate(
219
+ subject=subj[:80],
220
+ relation=_safe_relation(f["relation"]),
221
+ value=str(f["value"])[:160],
222
+ confidence=conf,
223
+ pattern="llm_enrichment"))
224
+ if not cands:
225
+ continue
226
+ n = writer.ingest_candidates(
227
+ cands, user_id=uid,
228
+ agent_id=chunk.get("agent_id"), chunk_id=chunk["id"],
229
+ ts=_parse_ts(chunk.get("ts")), source="llm-enrichment",
230
+ extractor_model=model)
231
+ rep.facts_extracted += len(cands)
232
+ rep.facts_committed += n
233
+ rep.extractor_model = getattr(extractor, "model_name", None)
234
+ rep.seconds = round(time.time() - t0, 3)
235
+ return rep
236
+
237
+
238
+ def _safe_relation(rel) -> str:
239
+ r = normalize(str(rel)).replace(" ", "_").replace("-", "_")
240
+ return "".join(ch for ch in r if ch.isalnum() or ch == "_")[:40] or "related_to"
241
+
242
+
243
+ def enrich_async(writer: MemoryWriter, user_id: str | None = None, **kw):
244
+ """Fire-and-forget background enrichment. Returns (thread, result holder)."""
245
+ holder: dict = {}
246
+
247
+ def _run():
248
+ try:
249
+ holder["report"] = enrich(writer, user_id, **kw)
250
+ except Exception as e: # pragma: no cover - background guard
251
+ holder["error"] = str(e)
252
+
253
+ t = threading.Thread(target=_run, daemon=True, name="cortexm-enrich")
254
+ t.start()
255
+ return t, holder