secondbrain-py 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. brain/__init__.py +0 -0
  2. brain/__main__.py +18 -0
  3. brain/_capture_command.py +445 -0
  4. brain/_compose.py +52 -0
  5. brain/activity.py +206 -0
  6. brain/ask.py +631 -0
  7. brain/audio.py +591 -0
  8. brain/backfill/__init__.py +12 -0
  9. brain/backfill/search_extras.py +141 -0
  10. brain/backfill/source_rows.py +101 -0
  11. brain/bin/__init__.py +1 -0
  12. brain/bin/_launcher.py +107 -0
  13. brain/bin/down.py +8 -0
  14. brain/bin/launchd.py +268 -0
  15. brain/bin/monitor.py +570 -0
  16. brain/bin/rebuild.py +8 -0
  17. brain/bin/status.py +8 -0
  18. brain/bin/up.py +8 -0
  19. brain/brief.py +272 -0
  20. brain/capture.py +49 -0
  21. brain/chat.py +293 -0
  22. brain/cli.py +9760 -0
  23. brain/cli_claude.py +81 -0
  24. brain/cli_connect.py +285 -0
  25. brain/cli_demo.py +266 -0
  26. brain/config.py +1949 -0
  27. brain/connect.py +925 -0
  28. brain/db.py +540 -0
  29. brain/demo/__init__.py +452 -0
  30. brain/demo/corpus/manifest.json +403 -0
  31. brain/demo/embedder.py +74 -0
  32. brain/durations.py +84 -0
  33. brain/edit_session.py +156 -0
  34. brain/editor.py +67 -0
  35. brain/elicit/__init__.py +16 -0
  36. brain/elicit/detectors.py +250 -0
  37. brain/elicit/drafter.py +70 -0
  38. brain/elicit/queue.py +220 -0
  39. brain/elicit/schema.py +48 -0
  40. brain/elicit/session.py +445 -0
  41. brain/embedding_targets.py +54 -0
  42. brain/embeddings.py +424 -0
  43. brain/enrichment.py +808 -0
  44. brain/errors.py +357 -0
  45. brain/eval/__init__.py +129 -0
  46. brain/eval/answer_eval.py +281 -0
  47. brain/eval/baseline.py +265 -0
  48. brain/eval/concept_extraction.py +378 -0
  49. brain/eval/corpus.py +152 -0
  50. brain/eval/errors.py +19 -0
  51. brain/eval/graph_baseline.py +226 -0
  52. brain/eval/graph_retrieval.py +202 -0
  53. brain/eval/graph_runner.py +319 -0
  54. brain/eval/metrics.py +101 -0
  55. brain/eval/runner.py +223 -0
  56. brain/format.py +783 -0
  57. brain/gaps.py +390 -0
  58. brain/graph_rag/__init__.py +94 -0
  59. brain/graph_rag/_retrieval_common.py +113 -0
  60. brain/graph_rag/aggregates.py +303 -0
  61. brain/graph_rag/aliases/__init__.py +583 -0
  62. brain/graph_rag/backends/__init__.py +10 -0
  63. brain/graph_rag/backends/_age_helpers.py +473 -0
  64. brain/graph_rag/backends/age.py +782 -0
  65. brain/graph_rag/backends/base.py +272 -0
  66. brain/graph_rag/build.py +344 -0
  67. brain/graph_rag/communities.py +644 -0
  68. brain/graph_rag/communities_summary.py +437 -0
  69. brain/graph_rag/concepts.py +202 -0
  70. brain/graph_rag/cooccur.py +193 -0
  71. brain/graph_rag/cross_type.py +312 -0
  72. brain/graph_rag/extract.py +885 -0
  73. brain/graph_rag/fuse.py +371 -0
  74. brain/graph_rag/global_.py +412 -0
  75. brain/graph_rag/grouping.py +372 -0
  76. brain/graph_rag/person_resolver.py +167 -0
  77. brain/graph_rag/reconcile.py +792 -0
  78. brain/graph_rag/relational.py +353 -0
  79. brain/graph_rag/retrieve.py +526 -0
  80. brain/graph_rag/router.py +288 -0
  81. brain/graph_rag/schema.py +320 -0
  82. brain/graph_rag/sync.py +237 -0
  83. brain/graph_rag/tenancy.py +43 -0
  84. brain/graph_rag/themes.py +501 -0
  85. brain/graph_rag/weighting.py +202 -0
  86. brain/ingest/__init__.py +1926 -0
  87. brain/ingest/chunker.py +249 -0
  88. brain/ingest/docx.py +40 -0
  89. brain/ingest/gmail.py +621 -0
  90. brain/ingest/markdown.py +37 -0
  91. brain/ingest/pdf.py +61 -0
  92. brain/ingest/stdin.py +22 -0
  93. brain/ingest/sub_tokens.py +91 -0
  94. brain/ingest/text.py +16 -0
  95. brain/interactions.py +205 -0
  96. brain/maintenance.py +355 -0
  97. brain/mcp_server.py +3405 -0
  98. brain/migrations/001_init.sql +43 -0
  99. brain/migrations/002_qwen3_embedding.sql +17 -0
  100. brain/migrations/003_vault_model.sql +41 -0
  101. brain/migrations/004_relax_content_hash_uniqueness.sql +18 -0
  102. brain/migrations/005_derived_links.sql +67 -0
  103. brain/migrations/006_dedup_file_by_source_path.sql +25 -0
  104. brain/migrations/007_email_thread_and_draft.sql +15 -0
  105. brain/migrations/008_gmail_thread_unique.sql +11 -0
  106. brain/migrations/009_chunks_weighted_tsv.sql +28 -0
  107. brain/migrations/010_interactions.sql +30 -0
  108. brain/migrations/011_documents_summary.sql +23 -0
  109. brain/migrations/012_graphrag.sql +171 -0
  110. brain/migrations/013_graphrag_communities.sql +125 -0
  111. brain/migrations/014_graphrag_community_summary_hash.sql +33 -0
  112. brain/migrations/015_interactions_graph_targets.sql +89 -0
  113. brain/migrations/016_index_hygiene.sql +61 -0
  114. brain/migrations/017_elicit.sql +30 -0
  115. brain/migrations/018_review_gap_signal_kinds.sql +40 -0
  116. brain/migrations/019_search_queries.sql +35 -0
  117. brain/migrations/020_link_suggestions.sql +40 -0
  118. brain/migrations/021_timeline_doc_date.sql +34 -0
  119. brain/migrations/022_link_suggestions_undirected.sql +84 -0
  120. brain/migrations/023_search_queries_fts_count.sql +28 -0
  121. brain/quartz_overrides/__init__.py +8 -0
  122. brain/quartz_overrides/quartz/bootstrap-cli.mjs +65 -0
  123. brain/quartz_overrides/quartz/build.ts +568 -0
  124. brain/quartz_overrides/quartz/cli/args.js +152 -0
  125. brain/quartz_overrides/quartz/cli/build_partial_handler.js +544 -0
  126. brain/quartz_overrides/quartz/cli/handlers.js +636 -0
  127. brain/quartz_overrides/quartz/components/CommandPalette.tsx +172 -0
  128. brain/quartz_overrides/quartz/components/Explorer.tsx +198 -0
  129. brain/quartz_overrides/quartz/components/Footer.tsx +27 -0
  130. brain/quartz_overrides/quartz/components/Graph.tsx +468 -0
  131. brain/quartz_overrides/quartz/components/PageTitle.tsx +72 -0
  132. brain/quartz_overrides/quartz/components/RelatedDocs.tsx +38 -0
  133. brain/quartz_overrides/quartz/components/Search.tsx +161 -0
  134. brain/quartz_overrides/quartz/components/SummaryLede.tsx +72 -0
  135. brain/quartz_overrides/quartz/components/index.ts +92 -0
  136. brain/quartz_overrides/quartz/components/pages/TagContent.tsx +272 -0
  137. brain/quartz_overrides/quartz/components/scripts/commandPalette.inline.ts +665 -0
  138. brain/quartz_overrides/quartz/components/scripts/explorer.inline.ts +768 -0
  139. brain/quartz_overrides/quartz/components/scripts/graph.inline.ts +2302 -0
  140. brain/quartz_overrides/quartz/components/scripts/relatedDocs.inline.ts +163 -0
  141. brain/quartz_overrides/quartz/components/scripts/search.inline.ts +1011 -0
  142. brain/quartz_overrides/quartz/plugins/emitters/contentIndex.ts +546 -0
  143. brain/quartz_overrides/quartz/plugins/transformers/codeCopy.ts +94 -0
  144. brain/quartz_overrides/quartz/plugins/transformers/derivedFenceMark.ts +302 -0
  145. brain/quartz_overrides/quartz/plugins/transformers/emailThread.ts +148 -0
  146. brain/quartz_overrides/quartz/plugins/transformers/emptyDoorFilter.ts +213 -0
  147. brain/quartz_overrides/quartz/plugins/transformers/index.ts +114 -0
  148. brain/quartz_overrides/quartz/plugins/transformers/linkKindMark.ts +205 -0
  149. brain/quartz_overrides/quartz/plugins/transformers/linkSourceTag.ts +104 -0
  150. brain/quartz_overrides/quartz/plugins/transformers/relativeDate.ts +100 -0
  151. brain/quartz_overrides/quartz/plugins/transformers/reloadSignal.ts +131 -0
  152. brain/quartz_overrides/quartz/processors/parse.ts +371 -0
  153. brain/quartz_overrides/quartz/processors/parser_cache.ts +78 -0
  154. brain/quartz_overrides/quartz/static/brain-logo-dark.png +0 -0
  155. brain/quartz_overrides/quartz/static/brain-logo-light.png +0 -0
  156. brain/quartz_overrides/quartz/static/codeCopy.js +196 -0
  157. brain/quartz_overrides/quartz/static/emailThread.js +334 -0
  158. brain/quartz_overrides/quartz/static/favicon.ico +0 -0
  159. brain/quartz_overrides/quartz/static/icon.png +0 -0
  160. brain/quartz_overrides/quartz/static/linkSourceTag.js +104 -0
  161. brain/quartz_overrides/quartz/static/relativeDate.js +142 -0
  162. brain/quartz_overrides/quartz/static/reload.js +168 -0
  163. brain/quartz_overrides/quartz/styles/brain/_article.scss +252 -0
  164. brain/quartz_overrides/quartz/styles/brain/_atmosphere.scss +113 -0
  165. brain/quartz_overrides/quartz/styles/brain/_callouts.scss +180 -0
  166. brain/quartz_overrides/quartz/styles/brain/_cmdk.scss +7 -0
  167. brain/quartz_overrides/quartz/styles/brain/_code.scss +208 -0
  168. brain/quartz_overrides/quartz/styles/brain/_command_palette.scss +369 -0
  169. brain/quartz_overrides/quartz/styles/brain/_email_thread.scss +228 -0
  170. brain/quartz_overrides/quartz/styles/brain/_explorer.scss +142 -0
  171. brain/quartz_overrides/quartz/styles/brain/_home.scss +182 -0
  172. brain/quartz_overrides/quartz/styles/brain/_links.scss +322 -0
  173. brain/quartz_overrides/quartz/styles/brain/_marginalia.scss +117 -0
  174. brain/quartz_overrides/quartz/styles/brain/_motion.scss +175 -0
  175. brain/quartz_overrides/quartz/styles/brain/_people_hub.scss +100 -0
  176. brain/quartz_overrides/quartz/styles/brain/_related_docs.scss +137 -0
  177. brain/quartz_overrides/quartz/styles/brain/_search.scss +252 -0
  178. brain/quartz_overrides/quartz/styles/brain/_sidebar.scss +468 -0
  179. brain/quartz_overrides/quartz/styles/brain/_summary_lede.scss +56 -0
  180. brain/quartz_overrides/quartz/styles/brain/_surface.scss +43 -0
  181. brain/quartz_overrides/quartz/styles/brain/_tag_content.scss +118 -0
  182. brain/quartz_overrides/quartz/styles/brain/_tokens.scss +197 -0
  183. brain/quartz_overrides/quartz/styles/brain/_typography.scss +92 -0
  184. brain/quartz_overrides/quartz/styles/custom.scss +89 -0
  185. brain/quartz_overrides/quartz/styles/graph.scss +505 -0
  186. brain/quartz_overrides/quartz/util/ctx.ts +92 -0
  187. brain/quartz_overrides/quartz/util/fastpath_manifest.ts +608 -0
  188. brain/quartz_overrides/quartz/util/path.ts +358 -0
  189. brain/quartz_overrides/quartz/util/sourceIcons.ts +55 -0
  190. brain/quartz_overrides/quartz.config.ts +270 -0
  191. brain/quartz_overrides/quartz.layout.ts +314 -0
  192. brain/queries.py +1188 -0
  193. brain/rank_fusion.py +8 -0
  194. brain/resurface.py +210 -0
  195. brain/review/__init__.py +26 -0
  196. brain/review/emit.py +27 -0
  197. brain/review/queries.py +436 -0
  198. brain/review/render.py +196 -0
  199. brain/review/scans.py +355 -0
  200. brain/review/weekly.py +413 -0
  201. brain/search.py +704 -0
  202. brain/set_similarity.py +15 -0
  203. brain/setup.py +1205 -0
  204. brain/tags.py +56 -0
  205. brain/templates/Caddyfile.j2 +9 -0
  206. brain/templates/__init__.py +1 -0
  207. brain/templates/bin/__init__.py +1 -0
  208. brain/templates/bin/_brain-brief-fg.sh +25 -0
  209. brain/templates/bin/_brain-build-fg.sh +53 -0
  210. brain/templates/bin/_brain-watcher-fg.sh +65 -0
  211. brain/templates/bin/brain-down.sh +89 -0
  212. brain/templates/bin/brain-status.sh +83 -0
  213. brain/templates/bin/brain-up.sh +221 -0
  214. brain/templates/docker/age/Dockerfile +79 -0
  215. brain/templates/docker-compose.stock.yml.j2 +26 -0
  216. brain/templates/docker-compose.yml.j2 +34 -0
  217. brain/templates/env.example +190 -0
  218. brain/templates/launchd/__init__.py +1 -0
  219. brain/templates/launchd/com.brain.brief.plist.j2 +45 -0
  220. brain/templates/launchd/com.brain.build.plist.j2 +46 -0
  221. brain/templates/launchd/com.brain.watcher.plist.j2 +46 -0
  222. brain/templates/skill/SKILL.md +63 -0
  223. brain/templates/skill/__init__.py +1 -0
  224. brain/timeline.py +834 -0
  225. brain/todo.py +124 -0
  226. brain/uninstall.py +185 -0
  227. brain/vault/__init__.py +115 -0
  228. brain/vault/_atomic.py +25 -0
  229. brain/vault/daily_index.py +228 -0
  230. brain/vault/derived_links/__init__.py +50 -0
  231. brain/vault/derived_links/directory.py +683 -0
  232. brain/vault/derived_links/fence.py +408 -0
  233. brain/vault/derived_links/gws.py +64 -0
  234. brain/vault/derived_links/participants.py +143 -0
  235. brain/vault/derived_links/pass_runner.py +362 -0
  236. brain/vault/derived_links/rules.py +137 -0
  237. brain/vault/export.py +683 -0
  238. brain/vault/frontmatter.py +165 -0
  239. brain/vault/graph.py +620 -0
  240. brain/vault/graph_format.py +388 -0
  241. brain/vault/link_rewrite.py +235 -0
  242. brain/vault/links.py +260 -0
  243. brain/vault/note_builder.py +211 -0
  244. brain/vault/paths.py +55 -0
  245. brain/vault/quartz_overlay.py +236 -0
  246. brain/vault/rename.py +591 -0
  247. brain/vault/resolver.py +304 -0
  248. brain/vault/slug.py +127 -0
  249. brain/vault/sync.py +1513 -0
  250. brain/vault/sync_summaries.py +264 -0
  251. brain/vault/templates.py +145 -0
  252. brain/vault/watch.py +1052 -0
  253. brain/wiki/__init__.py +6 -0
  254. brain/wiki/_github_slugger.py +76 -0
  255. brain/wiki/_person_name.py +314 -0
  256. brain/wiki/build_homepage.py +541 -0
  257. brain/wiki/build_partial.py +273 -0
  258. brain/wiki/build_people.py +934 -0
  259. brain/wiki/build_related.py +758 -0
  260. brain/wiki/build_swap.py +585 -0
  261. brain/wiki/build_watcher.py +975 -0
  262. brain/wiki/edit_classifier.py +215 -0
  263. brain/wiki/errors.py +10 -0
  264. brain/wiki/fastpath_manifest.py +475 -0
  265. brain/wiki/fastpath_state.py +174 -0
  266. brain/wiki/install.py +296 -0
  267. brain/wiki/slug.py +111 -0
  268. secondbrain_py-0.2.1.dist-info/METADATA +195 -0
  269. secondbrain_py-0.2.1.dist-info/RECORD +273 -0
  270. secondbrain_py-0.2.1.dist-info/WHEEL +5 -0
  271. secondbrain_py-0.2.1.dist-info/entry_points.txt +11 -0
  272. secondbrain_py-0.2.1.dist-info/licenses/LICENSE +21 -0
  273. secondbrain_py-0.2.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1926 @@
1
+ """Ingest pipeline: extract → chunk → embed → store."""
2
+ from __future__ import annotations
3
+
4
+ import hashlib
5
+ import json
6
+ import logging
7
+ from dataclasses import dataclass
8
+ from datetime import UTC, datetime, timedelta
9
+ from email.utils import parsedate_to_datetime
10
+ from pathlib import Path
11
+ from typing import TYPE_CHECKING, Any, Protocol
12
+
13
+ import psycopg
14
+ from pgvector.psycopg import register_vector # noqa: F401 (ensures adapter loaded)
15
+
16
+ from brain.errors import EnrichmentError, IngestAmbiguousSource, OllamaUnavailable
17
+ from brain.tags import normalize_tags
18
+
19
+ if TYPE_CHECKING:
20
+ from brain.enrichment import OllamaEnricher
21
+ from brain.graph_rag.sync import GraphSyncer
22
+ from brain.vault.derived_links.directory import (
23
+ DirectoryStore,
24
+ GwsRunner,
25
+ refresh_calendar,
26
+ refresh_contacts,
27
+ )
28
+ from brain.vault.derived_links.participants import (
29
+ extract_gmail_addresses,
30
+ extract_krisp_speakers,
31
+ )
32
+ from brain.vault.export import regenerate_vault_file
33
+
34
+ from .chunker import Chunk, chunk_text
35
+ from .sub_tokens import extract_sub_tokens
36
+
37
+ _logger = logging.getLogger(__name__)
38
+
39
+ # Contacts refresh is rate-limited to once per 24 hours. Krisp ingest is the
40
+ # trigger; without this gate, every transcript would re-fetch the full Google
41
+ # People page.
42
+ _CONTACTS_REFRESH_INTERVAL = timedelta(hours=24)
43
+
44
+
45
+ class Embedder(Protocol):
46
+ """Narrow interface for embedding clients used by the ingest pipeline.
47
+
48
+ ``dim`` is the embedder's native output dimension. Schema-wiring code
49
+ (``db.ensure_embedding_column``, ``queries.finalize_embedding_index``)
50
+ reads it to keep the ``chunks.embedding`` column in lockstep with the
51
+ active backend, so callers stay backend-agnostic.
52
+ """
53
+
54
+ dim: int
55
+
56
+ def embed(
57
+ self, texts: list[str], *, input_type: str = "document"
58
+ ) -> list[list[float]]: ...
59
+
60
+ def count_tokens(self, text: str) -> int: ...
61
+
62
+
63
+ @dataclass
64
+ class ExtractedDoc:
65
+ """A document produced by an extractor and ready to be ingested."""
66
+
67
+ title: str
68
+ content: str
69
+ content_type: str
70
+ source_path: str | None
71
+ metadata: dict[str, Any]
72
+
73
+
74
+ @dataclass
75
+ class IngestResult:
76
+ """Outcome of :func:`ingest_document`.
77
+
78
+ ``body_changed`` is ``True`` iff ``existing_hash != incoming_hash`` at the
79
+ moment the in-place UPDATE was applied. Concretely:
80
+
81
+ - New document (``created=True``): always ``True``.
82
+ - In-place UPDATE (``created=False``) where the stored content_hash
83
+ differs from the incoming hash: ``True`` — body was rewritten.
84
+ - In-place UPDATE where hashes match (same body, ``--force`` or changed
85
+ tags): ``False`` — no body change even though chunks were mechanically
86
+ rebuilt.
87
+ - No-op short-circuit (same hash, no ``--force``): ``False``.
88
+
89
+ Note: ``_update_doc_in_place`` always rebuilds chunks (DELETE + INSERT)
90
+ regardless of ``body_changed`` — chunk row UUIDs change on every forced
91
+ re-ingest, but ``body_changed`` reports the *content-hash identity*, not
92
+ the chunk-row machinery. The mirror trigger in :func:`ingest_document`
93
+ reads ``body_changed OR force`` so an in-place update with only tag changes
94
+ still propagates to disk.
95
+ """
96
+
97
+ document_id: str | None
98
+ created: bool
99
+ body_changed: bool = False
100
+ # Wave G1-c: set ONLY on the force content-hash-fallback path
101
+ # (:func:`_ingest_within_transaction`), where an old ``documents`` row at
102
+ # the same content hash is DELETEd and a fresh row INSERTed under a NEW
103
+ # uuid. The old uuid's relational graph rows cascade away with the row, but
104
+ # its AGE ``Document`` vertex (and any persons it orphaned) do NOT — so
105
+ # :func:`ingest_document` calls ``graph_syncer.remove(replaced_document_id)``
106
+ # post-commit to drop the stale graph presence. ``None`` on every other
107
+ # path (new insert, in-place update, no-op skip).
108
+ replaced_document_id: str | None = None
109
+
110
+
111
+ @dataclass
112
+ class UpdateResult:
113
+ """Outcome of :func:`update_document`.
114
+
115
+ ``fields_changed`` lists the document columns that were actually mutated
116
+ (subset of ``{"title", "content", "content_type", "metadata", "tags"}``).
117
+ ``rechunked`` is ``True`` iff the body was replaced and chunks were
118
+ re-embedded.
119
+ """
120
+
121
+ document_id: str
122
+ fields_changed: list[str]
123
+ rechunked: bool
124
+
125
+
126
+ def _content_hash(text: str) -> str:
127
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
128
+
129
+
130
+ def _chunk_search_metadata(title: str, tags: list[str]) -> tuple[str, str]:
131
+ """Compute the per-document ``(title_text, tags_text)`` chunk columns.
132
+
133
+ Migration 009 splits chunks.tsv into a weighted multi-field tsvector — the
134
+ title goes under weight ``A`` and the tags under weight ``B``. Both are
135
+ denormalized onto every chunk row so the FTS rewrite stays a single
136
+ GIN-indexable expression. This helper centralizes the projection so all
137
+ four ``INSERT INTO chunks`` sites + the ``sync_chunk_search_metadata``
138
+ helper in :mod:`brain.queries` agree on the exact textual form.
139
+
140
+ ``tags_text`` mirrors the SQL backfill pattern
141
+ ``array_to_string(d.tags, ' ')`` — space-joined, empty string when the
142
+ tag list is empty (NOT ``None``; we never write SQL NULL here so the
143
+ coalesce inside the generated tsv stays a no-op rather than a noisy
144
+ cast).
145
+ """
146
+ return title, " ".join(tags) if tags else ""
147
+
148
+
149
+ def _is_gmail_thread_doc(doc: ExtractedDoc, source_kind: str) -> bool:
150
+ """Return True iff ``doc`` is a P2.1 merged-thread Gmail doc.
151
+
152
+ The marker shape — ``source_kind == "gmail"`` AND
153
+ ``content_type == "email_thread"`` AND ``metadata.thread_id`` populated
154
+ with a non-empty string — is the same triple that the partial unique
155
+ index ``uq_documents_gmail_thread`` (migration 008) constrains. Anything
156
+ else (legacy ``content_type == "email"`` per-message rows, manual
157
+ ``email_thread`` docs without a thread_id) falls through to the legacy
158
+ content_hash dedup path. Centralising the check here keeps the call
159
+ sites in :func:`ingest_document` and :func:`_ingest_within_transaction`
160
+ in lockstep — divergence between them would silently break the upsert.
161
+ """
162
+ thread_id = doc.metadata.get("thread_id")
163
+ return (
164
+ source_kind == "gmail"
165
+ and doc.content_type == "email_thread"
166
+ and isinstance(thread_id, str)
167
+ and bool(thread_id)
168
+ )
169
+
170
+
171
+ # Metadata keys promoted into typed columns on ``documents``. The mapping is
172
+ # intentionally narrow: every key here must correspond to a column added by
173
+ # migration 007. Anything outside this set stays in the JSONB ``metadata``
174
+ # blob, untouched.
175
+ _PROMOTED_COLUMNS: tuple[str, ...] = (
176
+ "thread_id",
177
+ "rfc_message_id",
178
+ "in_reply_to",
179
+ "sent_at",
180
+ "participants",
181
+ "duration_min",
182
+ )
183
+
184
+
185
+ def _parse_sent_at(raw: Any) -> datetime | None:
186
+ """Parse a date string into a TZ-aware UTC ``datetime``.
187
+
188
+ Accepts both RFC 2822 (Gmail ``Date:`` header style — e.g.
189
+ ``"Tue, 04 May 2026 14:23:01 -0400"``) and ISO 8601 (Krisp-style — e.g.
190
+ ``"2026-05-04T14:23:01+00:00"``) inputs. Returns ``None`` for any input
191
+ we can't parse — callers log + skip the column rather than crashing the
192
+ ingest. Naive datetimes are treated as UTC.
193
+ """
194
+ if not isinstance(raw, str) or not raw.strip():
195
+ return None
196
+
197
+ parsed: datetime | None = None
198
+ # RFC 2822 first — ``parsedate_to_datetime`` is strict-but-tolerant and
199
+ # is the format Gmail's ``Date:`` header uses, so it's the more common
200
+ # case for this codebase.
201
+ try:
202
+ parsed = parsedate_to_datetime(raw)
203
+ except (TypeError, ValueError):
204
+ parsed = None
205
+ if parsed is None:
206
+ try:
207
+ # ``fromisoformat`` accepts the trailing-Z form in 3.11+.
208
+ parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
209
+ except ValueError:
210
+ return None
211
+
212
+ if parsed.tzinfo is None:
213
+ parsed = parsed.replace(tzinfo=UTC)
214
+ return parsed.astimezone(UTC)
215
+
216
+
217
+ def _promote_metadata_to_columns(metadata: dict[str, Any]) -> dict[str, Any]:
218
+ """Project a metadata blob onto the typed ``documents`` columns.
219
+
220
+ Returns a ``{column_name: value}`` dict — keys are limited to the columns
221
+ in :data:`_PROMOTED_COLUMNS`. Missing-or-``None`` metadata keys are
222
+ omitted (callers never need to write a NULL explicitly — the column
223
+ default of NULL handles it on INSERT, and on UPDATE we just leave the
224
+ column alone). Malformed values are logged at WARNING and skipped so a
225
+ bad header never blocks an ingest.
226
+
227
+ Type-coercion rules:
228
+
229
+ - ``thread_id`` / ``rfc_message_id`` / ``in_reply_to`` — accept ``str``;
230
+ anything else is logged + skipped.
231
+ - ``date`` (RFC 2822 or ISO 8601 string) → ``sent_at`` (TZ-aware UTC).
232
+ Unparseable strings log + skip; we never store a partially-parsed
233
+ datetime.
234
+ - ``participants`` — accept ``list[str]`` (Krisp speakers, Gmail headers
235
+ already-parsed by an upstream extractor). Non-list / mixed-type lists
236
+ log + skip rather than silently coercing.
237
+ - ``duration_min`` — accept ``int``, ``float``, or a numeric ``str``.
238
+ Floats — and float-valued strings like ``"42.7"`` — round down via
239
+ ``int(float(...))``; ``bool`` and non-numeric values log + skip.
240
+ """
241
+ out: dict[str, Any] = {}
242
+
243
+ for key in ("thread_id", "rfc_message_id", "in_reply_to"):
244
+ raw = metadata.get(key)
245
+ if raw is None:
246
+ continue
247
+ if isinstance(raw, str):
248
+ out[key] = raw
249
+ else:
250
+ _logger.warning(
251
+ "metadata.%s expected str, got %s; skipping column promotion",
252
+ key,
253
+ type(raw).__name__,
254
+ )
255
+
256
+ raw_date = metadata.get("date")
257
+ if raw_date is not None:
258
+ sent_at = _parse_sent_at(raw_date)
259
+ if sent_at is not None:
260
+ out["sent_at"] = sent_at
261
+ else:
262
+ _logger.warning(
263
+ "metadata.date is not parseable (%r); leaving sent_at NULL",
264
+ raw_date,
265
+ )
266
+
267
+ raw_participants = metadata.get("participants")
268
+ if raw_participants is not None:
269
+ if isinstance(raw_participants, list) and all(
270
+ isinstance(p, str) for p in raw_participants
271
+ ):
272
+ out["participants"] = list(raw_participants)
273
+ else:
274
+ _logger.warning(
275
+ "metadata.participants expected list[str], got %r; "
276
+ "skipping column promotion",
277
+ type(raw_participants).__name__,
278
+ )
279
+
280
+ raw_duration = metadata.get("duration_min")
281
+ if raw_duration is not None:
282
+ try:
283
+ # ``bool`` is a subclass of ``int`` in Python — reject it
284
+ # explicitly so ``True`` doesn't end up as ``1`` in the column.
285
+ if isinstance(raw_duration, bool):
286
+ raise TypeError("bool is not a valid duration")
287
+ # ``int(float(...))`` so numeric strings with a fractional part
288
+ # ("42.7", Krisp's occasional float-minute serialization) round
289
+ # down instead of raising ValueError under a bare ``int("42.7")``.
290
+ out["duration_min"] = int(float(raw_duration))
291
+ except (TypeError, ValueError):
292
+ _logger.warning(
293
+ "metadata.duration_min expected int, got %r; "
294
+ "skipping column promotion",
295
+ raw_duration,
296
+ )
297
+
298
+ return out
299
+
300
+
301
+ def _upsert_source(
302
+ conn: psycopg.Connection,
303
+ *,
304
+ kind: str,
305
+ external_id: str | None,
306
+ metadata: dict[str, Any],
307
+ ) -> str | None:
308
+ """Return an existing source row id, or insert a new one. Returns None for
309
+ purely manual ingests with no external id and no metadata.
310
+
311
+ Wave perf-T3 (F6): the common (non-NULL ``external_id``) path is a single
312
+ ``INSERT … ON CONFLICT (kind, external_id)`` round-trip instead of a
313
+ SELECT-then-INSERT pair. The ``DO UPDATE SET metadata = sources.metadata``
314
+ clause is a deliberate no-op self-assignment — it forces ``RETURNING id`` to
315
+ yield the row even on conflict (``DO NOTHING`` returns no row) while
316
+ preserving the existing row's metadata, exactly matching the legacy
317
+ behavior, which returned the existing id WITHOUT refreshing metadata.
318
+
319
+ The ``UNIQUE (kind, external_id)`` constraint (migration 001) treats NULLs
320
+ as distinct, so an ``ON CONFLICT`` target cannot match a NULL
321
+ ``external_id``. The rare manual-stdin-with-metadata case
322
+ (``external_id is None`` but ``metadata`` non-empty) keeps the explicit
323
+ lookup-then-insert so it still reuses an existing NULL-external_id row.
324
+ """
325
+ if external_id is None and not metadata:
326
+ return None
327
+ if external_id is None:
328
+ row = conn.execute(
329
+ "SELECT id FROM sources WHERE kind=%s AND external_id IS NULL",
330
+ (kind,),
331
+ ).fetchone()
332
+ if row:
333
+ return str(row[0])
334
+ new = conn.execute(
335
+ "INSERT INTO sources (kind, external_id, metadata) "
336
+ "VALUES (%s, NULL, %s) RETURNING id",
337
+ (kind, json.dumps(metadata)),
338
+ ).fetchone()
339
+ assert new is not None # RETURNING id always yields a row
340
+ return str(new[0])
341
+ row = conn.execute(
342
+ "INSERT INTO sources (kind, external_id, metadata) VALUES (%s, %s, %s) "
343
+ "ON CONFLICT (kind, external_id) DO UPDATE SET metadata = sources.metadata "
344
+ "RETURNING id",
345
+ (kind, external_id, json.dumps(metadata)),
346
+ ).fetchone()
347
+ assert row is not None # ON CONFLICT DO UPDATE always yields a row
348
+ return str(row[0])
349
+
350
+
351
+ def _embed_chunks(
352
+ embedder: Embedder, texts: list[str]
353
+ ) -> list[list[float] | None]:
354
+ """Embed ``texts``, or return NULL placeholders under a no-vector backend.
355
+
356
+ Under the FTS-only ``none`` backend (:class:`brain.embeddings.NullEmbedder`,
357
+ ``produces_embeddings = False``) the pipeline must store NULL embeddings
358
+ rather than call ``embed()`` (which raises :class:`~brain.errors.EmbedError`).
359
+ Duck-typed via ``getattr`` so the real backends (Arctic / Qwen3 / Voyage) —
360
+ which never declare the flag — embed normally with zero changes. Returns one
361
+ entry per input text: a real vector, or ``None`` (bound as SQL NULL by
362
+ :func:`_insert_chunks`; the column is nullable pre-finalize). An empty
363
+ ``texts`` makes no embed call and returns ``[]``.
364
+ """
365
+ if not getattr(embedder, "produces_embeddings", True):
366
+ return [None] * len(texts)
367
+ # Widen the concrete ``list[list[float]]`` to the nullable element type via
368
+ # ``extend`` (Iterable is covariant, so this stays mypy-clean) — the real
369
+ # backends only ever contribute non-NULL vectors here.
370
+ embeddings: list[list[float] | None] = []
371
+ embeddings.extend(embedder.embed(texts, input_type="document"))
372
+ return embeddings
373
+
374
+
375
+ def _insert_chunks(
376
+ conn: psycopg.Connection,
377
+ *,
378
+ document_id: str,
379
+ chunks: list[Chunk],
380
+ embeddings: list[list[float] | None],
381
+ title_text: str,
382
+ tags_text: str,
383
+ ) -> None:
384
+ """Bulk-insert chunk rows for ``document_id`` in a single round-trip.
385
+
386
+ Wave perf-T3 (F2/F7): replaces the per-chunk ``conn.execute`` loop with one
387
+ ``executemany`` over a pre-built row list. ``extract_sub_tokens`` is computed
388
+ while building the list (before any DB round-trip) instead of interleaved
389
+ with N inserts. The ``title_text`` / ``tags_text`` / ``search_extras``
390
+ columns feed the migration-009 weighted multi-field tsv exactly as the prior
391
+ loop did, so the stored rows are byte-for-byte identical. ``embeddings`` must
392
+ be precomputed by the caller (outside the transaction). No-op on empty
393
+ ``chunks``.
394
+ """
395
+ if not chunks:
396
+ return
397
+ rows = [
398
+ (
399
+ document_id,
400
+ c.index,
401
+ c.content,
402
+ emb,
403
+ title_text,
404
+ tags_text,
405
+ extract_sub_tokens(c.content),
406
+ )
407
+ for c, emb in zip(chunks, embeddings, strict=True)
408
+ ]
409
+ # psycopg3 exposes ``executemany`` on the cursor, not the connection. The
410
+ # cursor runs inside the caller's open transaction (same connection), so
411
+ # the bulk insert stays atomic with the surrounding document write.
412
+ with conn.cursor() as cur:
413
+ cur.executemany(
414
+ "INSERT INTO chunks (document_id, chunk_index, content, embedding, "
415
+ "title_text, tags_text, search_extras) "
416
+ "VALUES (%s, %s, %s, %s, %s, %s, %s)",
417
+ rows,
418
+ )
419
+
420
+
421
+ def ingest_document(
422
+ conn: psycopg.Connection,
423
+ *,
424
+ embedder: Embedder,
425
+ doc: ExtractedDoc,
426
+ source_kind: str | None = None,
427
+ source_external_id: str | None = None,
428
+ source_metadata: dict[str, Any] | None = None,
429
+ tags: list[str] | None = None,
430
+ force: bool = False,
431
+ gws_runner: GwsRunner | None = None,
432
+ vault_root: Path | None = None,
433
+ draft: bool = False,
434
+ enricher: OllamaEnricher | None = None,
435
+ enrich: bool = True,
436
+ enrich_min_tokens: int = 50,
437
+ graph_syncer: GraphSyncer | None = None,
438
+ ) -> IngestResult:
439
+ """Ingest a single extracted document.
440
+
441
+ Dedup rules (evaluated in order):
442
+
443
+ 1. **Gmail thread** — deduped by ``(thread_id, content_type='email_thread')``.
444
+ Re-ingest of the same thread UPDATEs the existing row in place (UUID
445
+ stable; links / derived_links / unresolved_links preserved).
446
+
447
+ 2. **File-based** (``doc.source_path`` is not ``None``) — deduped by
448
+ ``source_path`` (``WHERE source_path=%s AND kind='ingested'``). When a
449
+ match is found the row is UPDATEd in place: the document UUID is
450
+ preserved, tags are **union**-merged (existing ∪ incoming, insertion
451
+ order), and ``links`` / ``derived_links`` / ``unresolved_links``
452
+ referencing the UUID survive. Chunks are always rebuilt. When no
453
+ source_path match exists the doc is INSERTed as a new row directly —
454
+ migration 006 narrows the content_hash UNIQUE constraint to stdin-only
455
+ rows, so the INSERT cannot collide with another file at a different path.
456
+ ``force=True`` forces the UPDATE even when the content_hash is unchanged.
457
+
458
+ 3. **Sourced stdin** (``source_path`` is ``None``, ``source_external_id``
459
+ is not ``None`` — Krisp, Slack, ...) — deduped by
460
+ ``(source_kind, source_external_id)`` via a JOIN on ``sources``. When a
461
+ match is found the row is UPDATEd in place (same UUID / tag union /
462
+ link-preservation semantics as the file branch). When no sourced row
463
+ exists, control falls through to rule 4.
464
+
465
+ 4. **Content-hash fallback** — reached only when ``source_path is None``
466
+ (stdin ingests). Deduped by SHA-256 of ``doc.content`` scoped to
467
+ ``kind='ingested' AND source_path IS NULL``. Same-hash is a no-op;
468
+ ``force=True`` does DELETE + INSERT (new UUID).
469
+
470
+ Tags are **union**-merged on all UPDATE paths: existing curated tags are
471
+ preserved; incoming tags are added. The union is computed via
472
+ ``normalize_tags([*existing, *incoming])`` which is idempotent (casefold +
473
+ hyphenate) and preserves first-seen insertion order.
474
+
475
+ Sources are deduped by ``(kind, external_id)``. A repeat ingest pointing
476
+ at the same external id reuses the existing source row.
477
+
478
+ Source defaults: file-based ingests (``doc.source_path`` set) default to
479
+ ``source_kind="manual"`` with ``source_external_id=doc.source_path`` when
480
+ the caller does not pass them, so the resulting document carries a
481
+ ``source_id`` pointing at a ``sources`` row with ``kind="manual"``.
482
+ Without this, downstream consumers (vault-export frontmatter, the graph
483
+ filter chips) lose the manual-source signal. Explicit ``source_kind`` /
484
+ ``source_external_id`` arguments always win, so a file-based ingest can
485
+ still be tagged as e.g. ``krisp`` if needed. Stdin ingests must pass
486
+ ``source_kind`` explicitly — they get no default.
487
+
488
+ Source-specific side effects split by cost (Task 2.11): the Gmail directory
489
+ upsert runs IN-TRANSACTION via :func:`_run_source_hooks` (fast local write
490
+ with an atomic-rollback contract), while the Krisp calendar / contacts
491
+ refresh runs POST-COMMIT via :func:`_run_post_commit_source_hooks` because it
492
+ shells out to ``gws`` (multi-second) and must not hold the write
493
+ transaction. The ``gws_runner`` argument is only consulted by the Krisp hook;
494
+ passing ``None`` skips the calendar / contacts refresh with a logged warning
495
+ so callers without a runner wired (early CLI paths, tests) still succeed.
496
+
497
+ Vault mirror: when ``vault_root`` is supplied AND the call actually wrote
498
+ a row (``result.created`` or ``force``), the corresponding mirror file
499
+ under ``vault_root / _ingested/<source>/...md`` is regenerated via
500
+ :func:`brain.vault.export.regenerate_vault_file`. The call runs OUTSIDE
501
+ the DB transaction — a filesystem failure is logged at WARNING and
502
+ swallowed so a transient mirror error never rolls back a successful
503
+ ingest. Recovery is via ``brain vault export --force``. Library callers
504
+ (tests, internal pipelines) that don't care about the mirror omit
505
+ ``vault_root`` and get the legacy DB-only behavior unchanged. Named
506
+ ``vault_root`` to distinguish from the per-document
507
+ ``documents.vault_path`` relative path.
508
+
509
+ Wave Q1-D enrichment: when ``enrich=True`` (default) AND ``enricher`` is
510
+ not None, a post-ingest hook generates a 2-3 sentence summary via local
511
+ Ollama and writes it to ``documents.summary`` inside the same
512
+ transaction. Skip rules: caller passes ``enrich=False``, content is
513
+ shorter than ``enrich_min_tokens`` tokens, ``documents.summary`` is
514
+ already populated with the same ``content_hash``, or Ollama is
515
+ unavailable (logged at WARNING — the ingest still commits;
516
+ ``brain enrich --backfill`` picks the row up later). Passing
517
+ ``enricher=None`` with ``enrich=True`` is a no-op with a debug log
518
+ (library callers / tests that don't care about enrichment leave it
519
+ None).
520
+
521
+ Wave G1-c graph sync: when ``graph_syncer`` is supplied AND the call
522
+ actually wrote a row (``created`` / ``body_changed`` / ``force``), the
523
+ document's people-aspect graph is reconciled post-commit via
524
+ :meth:`brain.graph_rag.sync.GraphSyncer.reconcile` (reusing ``conn``,
525
+ OUTSIDE the transaction — like the vault mirror). It is best-effort and
526
+ never raises: a disabled flag, an AGE-absent DB, or any sync error is a
527
+ logged no-op that leaves the committed ingest intact. ``None`` (library
528
+ callers / tests) skips graph sync entirely.
529
+ """
530
+ if doc.source_path is not None:
531
+ if source_kind is None:
532
+ source_kind = "manual"
533
+ if source_external_id is None:
534
+ source_external_id = doc.source_path
535
+ if source_kind is None:
536
+ raise ValueError(
537
+ "source_kind is required when doc.source_path is None"
538
+ )
539
+
540
+ # Gmail-thread upsert (P2.2): a merged-thread doc keys on the stable
541
+ # Gmail ``threadId`` rather than a per-message ``messageId``. Mirroring
542
+ # that invariant onto the ``sources`` row keeps source dedup aligned
543
+ # with the new partial unique index ``uq_documents_gmail_thread`` —
544
+ # repeated thread-batched ingests reuse the same ``sources`` row instead
545
+ # of accumulating one per re-ingest. The override only fires when the
546
+ # caller actually produced a thread doc (``content_type='email_thread'``
547
+ # AND ``metadata.thread_id`` populated); legacy per-message gmail rows
548
+ # still pass message_id as ``source_external_id`` and are unaffected.
549
+ if _is_gmail_thread_doc(doc, source_kind):
550
+ thread_id = doc.metadata.get("thread_id")
551
+ # _is_gmail_thread_doc guarantees thread_id is a non-empty str.
552
+ assert isinstance(thread_id, str) and thread_id
553
+ source_external_id = thread_id
554
+
555
+ h = _content_hash(doc.content)
556
+ tags = tags or []
557
+ source_metadata = source_metadata or {}
558
+
559
+ result = _ingest_within_transaction(
560
+ conn,
561
+ embedder=embedder,
562
+ doc=doc,
563
+ source_kind=source_kind,
564
+ source_external_id=source_external_id,
565
+ source_metadata=source_metadata,
566
+ tags=tags,
567
+ force=force,
568
+ gws_runner=gws_runner,
569
+ content_hash=h,
570
+ draft=draft,
571
+ enricher=enricher,
572
+ enrich=enrich,
573
+ enrich_min_tokens=enrich_min_tokens,
574
+ )
575
+
576
+ # Mirror writes happen OUTSIDE the transaction so a filesystem error
577
+ # cannot roll back the DB ingest (CLAUDE.md: prefer recoverable drift
578
+ # over an aborted ingest — drift is exactly what `brain vault export`
579
+ # exists to reconcile). ``body_changed`` covers the gmail-thread
580
+ # in-place upsert case where ``created`` stays ``False`` but the body
581
+ # was rewritten — without it the on-disk mirror would lag the DB.
582
+ if (
583
+ vault_root is not None
584
+ and result.document_id is not None
585
+ and (result.created or result.body_changed or force)
586
+ ):
587
+ try:
588
+ # ``force=True`` because we just inserted/replaced the DB row —
589
+ # we know a change happened. Without it, the body-hash skip in
590
+ # ``_write_doc_file`` would mask any frontmatter-only delta
591
+ # (relevant when ``--force`` rewrites a doc with the same body
592
+ # but different metadata).
593
+ regenerate_vault_file(
594
+ conn, result.document_id, vault_path=vault_root, force=True
595
+ )
596
+ except OSError as exc:
597
+ # Only OSError is reachable here: ``regenerate_vault_file``'s
598
+ # ValueError paths (no document with id, kind='vault') are both
599
+ # impossible for a row we just inserted with default
600
+ # ``kind='ingested'``. Catching ValueError would mask unrelated
601
+ # bugs.
602
+ _logger.warning(
603
+ "vault mirror write failed for document %s: %s; "
604
+ "DB ingest succeeded — recover via `brain vault export`",
605
+ result.document_id,
606
+ exc,
607
+ )
608
+
609
+ # Wave G1-c — people-aspect graph sync. Runs OUTSIDE the DB transaction
610
+ # (post-commit, reusing ``conn``) on the same gate as the vault mirror:
611
+ # only when an actual write happened (created / body changed / forced).
612
+ # ``GraphSyncer.reconcile`` is best-effort and never raises — a graph-sync
613
+ # failure is logged and swallowed so a committed ingest is never undone
614
+ # (the graph is a recomputable mirror). A ``None`` syncer (library callers,
615
+ # tests) or a disabled / AGE-absent DB is a no-op.
616
+ if (
617
+ graph_syncer is not None
618
+ and result.document_id is not None
619
+ and (result.created or result.body_changed or force)
620
+ ):
621
+ graph_syncer.reconcile(conn, result.document_id)
622
+ # Force content-hash-fallback replaced an old row under a new uuid — drop
623
+ # the old uuid's stale AGE Document vertex + any persons it orphaned. Runs
624
+ # AFTER reconcile(new) so shared persons keep their vertex; best-effort /
625
+ # never-raises like every other graph-sync call.
626
+ if graph_syncer is not None and result.replaced_document_id is not None:
627
+ graph_syncer.remove(conn, result.replaced_document_id)
628
+
629
+ return result
630
+
631
+
632
+ @dataclass(frozen=True)
633
+ class _IngestAction:
634
+ """The write the ingest pipeline will perform, decided by a read-only pass.
635
+
636
+ Computed by :func:`_resolve_ingest_action` BEFORE embeddings are generated
637
+ so the slow Ollama embed call (and the write transaction) is skipped
638
+ entirely on the no-op ``skip`` path. ``existing_id`` is the target row for
639
+ ``skip`` / ``update`` / ``replace`` and ``None`` for ``insert``.
640
+ """
641
+
642
+ kind: str # "skip" | "update" | "insert" | "replace"
643
+ existing_id: str | None
644
+ body_changed: bool
645
+
646
+
647
+ def _resolve_ingest_action(
648
+ conn: psycopg.Connection,
649
+ *,
650
+ doc: ExtractedDoc,
651
+ source_kind: str,
652
+ source_external_id: str | None,
653
+ content_hash: str,
654
+ force: bool,
655
+ draft: bool,
656
+ is_thread: bool,
657
+ ) -> _IngestAction:
658
+ """Read-only dedup decision — runs OUTSIDE the write transaction.
659
+
660
+ Replicates the dedup ladder previously embedded inside
661
+ :func:`_ingest_within_transaction` (Gmail thread → file ``source_path`` →
662
+ sourced-stdin → content-hash fallback) but performs only SELECTs, so the
663
+ caller can short-circuit ``skip`` (no embed, no transaction, no enrich) and
664
+ defer the embed for ``update`` / ``insert`` / ``replace`` until after the
665
+ slow Ollama call has run unlocked (Wave perf-T3 / F1).
666
+
667
+ Decisions, mirroring the legacy branches exactly:
668
+
669
+ - Gmail thread: draft-only-on-published → ``skip`` (Q1-A partial-window
670
+ no-op); same-hash-no-force → ``skip``; else ``update``
671
+ (``body_changed = existing_hash != content_hash``); no row → ``insert``.
672
+ - File ``source_path``: same-hash-no-force → ``skip``; else ``update``; no
673
+ row → ``insert`` (never reaches the content-hash fallback — migration 006
674
+ scopes the content_hash UNIQUE index to ``source_path IS NULL``).
675
+ - Sourced-stdin: >1 rows → :class:`IngestAmbiguousSource`; same-hash-no-force
676
+ → ``skip``; else ``update``; no row → content-hash fallback.
677
+ - Content-hash fallback (stdin only): row + no force → ``skip``; row + force
678
+ → ``replace`` (DELETE old + INSERT new uuid); no row → ``insert``.
679
+
680
+ The sourced branch drops the legacy ``FOR UPDATE OF d`` lock — the
681
+ authoritative row lock is re-acquired inside :func:`_update_doc_in_place` in
682
+ the write transaction. As before, this pipeline is safe for sequential CLI
683
+ use only; a concurrent dual-INSERT of the same key still raises
684
+ IntegrityError on the partial unique indexes.
685
+ """
686
+ h = content_hash
687
+ if is_thread:
688
+ thread_id = doc.metadata.get("thread_id")
689
+ assert isinstance(thread_id, str) and thread_id # _is_gmail_thread_doc
690
+ existing = conn.execute(
691
+ "SELECT id, content_hash, draft FROM documents "
692
+ "WHERE thread_id=%s AND kind='ingested' AND content_type='email_thread'",
693
+ (thread_id,),
694
+ ).fetchone()
695
+ if existing is not None:
696
+ existing_id, existing_hash, existing_draft = existing
697
+ if draft and not existing_draft:
698
+ return _IngestAction("skip", str(existing_id), False)
699
+ if not force and existing_hash == h:
700
+ return _IngestAction("skip", str(existing_id), False)
701
+ return _IngestAction("update", str(existing_id), existing_hash != h)
702
+ return _IngestAction("insert", None, True)
703
+
704
+ if doc.source_path is not None:
705
+ existing = conn.execute(
706
+ "SELECT id, content_hash FROM documents "
707
+ "WHERE source_path=%s AND kind='ingested'",
708
+ (doc.source_path,),
709
+ ).fetchone()
710
+ if existing:
711
+ existing_id, existing_hash = existing
712
+ if not force and existing_hash == h:
713
+ return _IngestAction("skip", str(existing_id), False)
714
+ return _IngestAction("update", str(existing_id), existing_hash != h)
715
+ return _IngestAction("insert", None, True)
716
+
717
+ # source_path is None — sourced lookup first, then content-hash fallback.
718
+ if source_external_id is not None:
719
+ sourced_rows = conn.execute(
720
+ """
721
+ SELECT d.id, d.content_hash
722
+ FROM documents d
723
+ JOIN sources s ON d.source_id = s.id
724
+ WHERE s.kind = %s AND s.external_id = %s
725
+ """,
726
+ (source_kind, source_external_id),
727
+ ).fetchall()
728
+ if len(sourced_rows) > 1:
729
+ raise IngestAmbiguousSource(
730
+ f"Multiple documents share source ({source_kind!r}, "
731
+ f"{source_external_id!r}): "
732
+ f"{[str(r[0]) for r in sourced_rows]}. "
733
+ "Resolve manually before re-ingesting."
734
+ )
735
+ if sourced_rows:
736
+ existing_id, existing_hash = sourced_rows[0]
737
+ if not force and existing_hash == h:
738
+ return _IngestAction("skip", str(existing_id), False)
739
+ return _IngestAction("update", str(existing_id), existing_hash != h)
740
+
741
+ existing = conn.execute(
742
+ "SELECT id FROM documents "
743
+ "WHERE content_hash=%s AND kind='ingested' AND source_path IS NULL",
744
+ (h,),
745
+ ).fetchone()
746
+ if existing:
747
+ if not force:
748
+ return _IngestAction("skip", str(existing[0]), False)
749
+ return _IngestAction("replace", str(existing[0]), True)
750
+ return _IngestAction("insert", None, True)
751
+
752
+
753
+ def _insert_new_document(
754
+ conn: psycopg.Connection,
755
+ *,
756
+ doc: ExtractedDoc,
757
+ source_kind: str,
758
+ source_external_id: str | None,
759
+ source_metadata: dict[str, Any],
760
+ tags: list[str],
761
+ content_hash: str,
762
+ draft: bool,
763
+ chunks: list[Chunk],
764
+ embeddings: list[list[float] | None],
765
+ ) -> str:
766
+ """INSERT a brand-new document row + its chunks; return the new id.
767
+
768
+ Runs inside the caller's open transaction. Embeddings are PRECOMPUTED by the
769
+ caller (outside the transaction, Wave perf-T3 / F1), so this helper performs
770
+ no Ollama calls and the transaction holds no locks across network I/O.
771
+ Auto-summary enrichment AND the slow Krisp calendar/contacts refresh are NOT
772
+ run here — the caller invokes :func:`_enrich_post_ingest_hook` and
773
+ :func:`_run_post_commit_source_hooks` AFTER the commit (Task 2.11), so the
774
+ Krisp ``gws`` subprocess calls never hold the write transaction open. The
775
+ fast in-transaction Gmail directory upsert (:func:`_run_source_hooks`) DOES
776
+ run here, preserving its atomic-rollback contract.
777
+ """
778
+ source_id = _upsert_source(
779
+ conn,
780
+ kind=source_kind,
781
+ external_id=source_external_id,
782
+ metadata=source_metadata,
783
+ )
784
+
785
+ # Pre-insert: derive source-specific metadata fields (e.g. Krisp
786
+ # ``_participant_keys``) so they're stored alongside the doc row in one
787
+ # INSERT. The leading underscore flags the key as derived/internal.
788
+ _apply_pre_insert_metadata(doc, source_kind=source_kind)
789
+
790
+ # Project email/krisp metadata onto typed columns. The base INSERT always
791
+ # writes the same fixed columns; the promoted columns ride along as a
792
+ # dynamic suffix so a doc with no recognized metadata keys produces the
793
+ # exact same SQL as before migration 007.
794
+ promoted = _promote_metadata_to_columns(doc.metadata)
795
+ promoted_columns = list(promoted.keys())
796
+ promoted_values = [promoted[c] for c in promoted_columns]
797
+ extra_cols = "".join(f", {c}" for c in promoted_columns)
798
+ extra_placeholders = ", %s" * len(promoted_columns)
799
+
800
+ doc_row = conn.execute(
801
+ f"""
802
+ INSERT INTO documents (source_id, title, content, content_hash, content_type,
803
+ source_path, tags, metadata, draft{extra_cols})
804
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s{extra_placeholders})
805
+ RETURNING id
806
+ """,
807
+ (
808
+ source_id,
809
+ doc.title,
810
+ doc.content,
811
+ content_hash,
812
+ doc.content_type,
813
+ doc.source_path,
814
+ tags,
815
+ json.dumps(doc.metadata),
816
+ draft,
817
+ *promoted_values,
818
+ ),
819
+ ).fetchone()
820
+ assert doc_row is not None # RETURNING id always yields a row
821
+ document_id = str(doc_row[0])
822
+
823
+ title_text, tags_text = _chunk_search_metadata(doc.title, tags)
824
+ _insert_chunks(
825
+ conn,
826
+ document_id=document_id,
827
+ chunks=chunks,
828
+ embeddings=embeddings,
829
+ title_text=title_text,
830
+ tags_text=tags_text,
831
+ )
832
+
833
+ # Wave Q1-D — Krisp action-items docs always carry the ``action-items`` tag
834
+ # so ``brain list --tag action-items`` and ``brain todo`` work out of the
835
+ # box. Mechanical, not LLM-derived; never re-proposed by ``brain tag
836
+ # --auto`` (D19 + D20).
837
+ _maybe_autotag_action_items(conn, document_id=document_id, doc=doc, tags=tags)
838
+
839
+ # In-transaction source hook (Gmail directory upsert). Its rollback contract
840
+ # is intentional — a directory-write failure rolls the document back. The
841
+ # slow Krisp calendar/contacts refresh runs POST-COMMIT instead — see
842
+ # :func:`_ingest_within_transaction` (Task 2.11).
843
+ _run_source_hooks(
844
+ conn,
845
+ source_kind=source_kind,
846
+ doc=doc,
847
+ document_id=document_id,
848
+ )
849
+ return document_id
850
+
851
+
852
+ def _ingest_within_transaction(
853
+ conn: psycopg.Connection,
854
+ *,
855
+ embedder: Embedder,
856
+ doc: ExtractedDoc,
857
+ source_kind: str,
858
+ source_external_id: str | None,
859
+ source_metadata: dict[str, Any],
860
+ tags: list[str],
861
+ force: bool,
862
+ gws_runner: GwsRunner | None,
863
+ content_hash: str,
864
+ draft: bool = False,
865
+ enricher: OllamaEnricher | None = None,
866
+ enrich: bool = True,
867
+ enrich_min_tokens: int = 50,
868
+ ) -> IngestResult:
869
+ """Decide the dedup action, embed OUTSIDE the transaction, then write.
870
+
871
+ Wave perf-T3 (F1) restructures the ingest hot path so neither slow Ollama
872
+ call holds a DB lock:
873
+
874
+ 1. :func:`_resolve_ingest_action` runs the read-only dedup ladder. A
875
+ ``skip`` decision (same content_hash / draft partial-window no-op)
876
+ returns immediately — no embed, no transaction, no enrich (matches the
877
+ prior early-return short-circuits exactly).
878
+ 2. ``embedder.embed`` runs BEFORE ``with conn.transaction()`` opens, so the
879
+ 2–5 s Ollama embed never holds a row lock. Skipped when there are no
880
+ chunks. (Previously the in-place UPDATE path embedded while holding a
881
+ ``FOR UPDATE`` lock on the document row.)
882
+ 3. The transaction performs ONLY the INSERT/UPDATE of the document + its
883
+ chunks using the precomputed vectors — it commits in ~milliseconds.
884
+ 4. Auto-summary enrichment runs AFTER the commit, in autocommit mode, as
885
+ its own short ``UPDATE``. A post-commit enrichment failure leaves the
886
+ committed document with ``summary IS NULL`` (best-effort; recoverable via
887
+ ``brain enrich --backfill``) — it can never roll back the ingest.
888
+
889
+ Atomicity preserved: the document row + ALL its chunk rows still INSERT (or
890
+ UPDATE + re-chunk) inside ONE transaction, so a write failure rolls back the
891
+ whole document. An embed failure raises BEFORE the transaction opens (no
892
+ partial row, no orphaned chunks). Sequential-CLI dedup guarantees are
893
+ unchanged; the partial-window draft no-op is enforced in BOTH the decision
894
+ pass and inside :func:`_update_doc_in_place` (defense in depth).
895
+ """
896
+ h = content_hash
897
+ is_thread = _is_gmail_thread_doc(doc, source_kind)
898
+
899
+ action = _resolve_ingest_action(
900
+ conn,
901
+ doc=doc,
902
+ source_kind=source_kind,
903
+ source_external_id=source_external_id,
904
+ content_hash=h,
905
+ force=force,
906
+ draft=draft,
907
+ is_thread=is_thread,
908
+ )
909
+ if action.kind == "skip":
910
+ return IngestResult(
911
+ document_id=action.existing_id, created=False, body_changed=False
912
+ )
913
+
914
+ # Chunk (cheap, pure Python) then embed — BOTH outside the transaction so
915
+ # the Ollama HTTP round-trip holds no DB lock.
916
+ chunks = chunk_text(doc.content, count_tokens=embedder.count_tokens)
917
+ if action.kind in ("insert", "replace") and not chunks:
918
+ # Matches the legacy INSERT-path "no chunks → no document" behavior.
919
+ # (The update path still proceeds on empty chunks — it rebuilds zero
920
+ # chunk rows, preserving the prior in-place behavior.)
921
+ return IngestResult(document_id=None, created=False)
922
+ embeddings = _embed_chunks(embedder, [c.content for c in chunks])
923
+
924
+ replaced_id: str | None = None
925
+ did_write = True
926
+ with conn.transaction():
927
+ if action.kind == "update":
928
+ assert action.existing_id is not None
929
+ did_write = _update_doc_in_place(
930
+ conn,
931
+ document_id=action.existing_id,
932
+ doc=doc,
933
+ source_kind=source_kind,
934
+ source_external_id=source_external_id,
935
+ source_metadata=source_metadata,
936
+ tags=tags,
937
+ content_hash=h,
938
+ body_changed=action.body_changed,
939
+ draft=draft,
940
+ chunks=chunks,
941
+ embeddings=embeddings,
942
+ )
943
+ document_id = action.existing_id
944
+ created = False
945
+ # When the partial-window draft guard no-op'd, nothing was written.
946
+ body_changed = action.body_changed if did_write else False
947
+ else: # "insert" or "replace"
948
+ if action.kind == "replace":
949
+ assert action.existing_id is not None
950
+ # Force re-ingest: replace the old row under a new uuid. Capture
951
+ # the old id so the post-commit graph hook can DETACH DELETE its
952
+ # now-orphaned AGE vertex (relational graph rows cascade with the
953
+ # row; AGE doesn't).
954
+ replaced_id = action.existing_id
955
+ conn.execute(
956
+ "DELETE FROM documents WHERE id=%s", (action.existing_id,)
957
+ )
958
+ document_id = _insert_new_document(
959
+ conn,
960
+ doc=doc,
961
+ source_kind=source_kind,
962
+ source_external_id=source_external_id,
963
+ source_metadata=source_metadata,
964
+ tags=tags,
965
+ content_hash=h,
966
+ draft=draft,
967
+ chunks=chunks,
968
+ embeddings=embeddings,
969
+ )
970
+ created = True
971
+ body_changed = True
972
+
973
+ # --- Post-commit Krisp source hook (Task 2.11). ---
974
+ # The Krisp calendar+contacts refresh runs AFTER the document transaction
975
+ # commits, on the same ``did_write`` gate the enrich step uses. Previously it
976
+ # ran INSIDE ``conn.transaction()``, so the hook held the write transaction
977
+ # open across multi-second ``gws`` subprocess calls (calendar + contacts
978
+ # refresh). Moving it post-commit loses no rollback semantics: both refresh
979
+ # helpers soft-degrade (catch runner failures and return 0). The Gmail
980
+ # directory upsert deliberately stays IN-TRANSACTION (fast local write with
981
+ # an atomic-rollback contract) — it already ran inside the transaction above.
982
+ # ``did_write`` is False only on the partial-window draft no-op, which wrote
983
+ # nothing — matching the prior in-transaction gating.
984
+ if did_write:
985
+ _run_post_commit_source_hooks(
986
+ conn,
987
+ source_kind=source_kind,
988
+ doc=doc,
989
+ document_id=document_id,
990
+ gws_runner=gws_runner,
991
+ )
992
+
993
+ # --- Post-commit auto-summary (Wave perf-T3 / F1). ---
994
+ # Runs in autocommit (the transaction has committed) as its own short
995
+ # UPDATE. ``is_new_doc=created`` skips the redundant idempotency SELECT on
996
+ # the create path (F4) — a freshly-INSERTed row is known to have
997
+ # ``summary IS NULL``. The update path keeps the SELECT for D11 idempotency.
998
+ # ``did_write`` is False only when :func:`_update_doc_in_place` hit its
999
+ # partial-window draft guard, in which case no body was written and we must
1000
+ # not enrich. Skip rules + failure handling (Ollama down → WARN, ingest
1001
+ # already committed) live inside the hook.
1002
+ if did_write:
1003
+ _enrich_post_ingest_hook(
1004
+ conn,
1005
+ document_id=document_id,
1006
+ doc=doc,
1007
+ enricher=enricher,
1008
+ enrich=enrich,
1009
+ min_tokens=enrich_min_tokens,
1010
+ content_hash=h,
1011
+ body_changed=body_changed,
1012
+ is_new_doc=created,
1013
+ )
1014
+
1015
+ return IngestResult(
1016
+ document_id=document_id,
1017
+ created=created,
1018
+ body_changed=body_changed,
1019
+ replaced_document_id=replaced_id,
1020
+ )
1021
+
1022
+
1023
+ def _update_doc_in_place(
1024
+ conn: psycopg.Connection,
1025
+ *,
1026
+ document_id: str,
1027
+ doc: ExtractedDoc,
1028
+ source_kind: str,
1029
+ source_external_id: str | None,
1030
+ source_metadata: dict[str, Any],
1031
+ tags: list[str],
1032
+ content_hash: str,
1033
+ body_changed: bool,
1034
+ chunks: list[Chunk],
1035
+ embeddings: list[list[float] | None],
1036
+ draft: bool = False,
1037
+ ) -> bool:
1038
+ """Replace title / body / metadata / tags / typed columns on an existing
1039
+ document row and rebuild its chunks. Returns ``True`` when the row was
1040
+ written, ``False`` when the partial-window draft guard no-op'd.
1041
+
1042
+ Used by the gmail-thread upsert path (P2.2), the file-path UPDATE-in-place
1043
+ branch, and the sourced-stdin (Krisp/Slack) UPDATE-in-place branch. Runs
1044
+ inside the caller's open transaction. The document UUID is preserved so any
1045
+ ``links`` / ``derived_links`` / ``unresolved_links`` referencing this doc
1046
+ keep pointing at the same row across re-ingest. Old chunks are deleted via
1047
+ direct DELETE (no need for ON DELETE CASCADE — we know the parent doc id)
1048
+ and re-inserted from the precomputed vectors. The Gmail directory upsert
1049
+ re-runs IN-TRANSACTION so headers from the latest message in the rebuilt
1050
+ thread propagate (its atomic-rollback contract is preserved); the slow Krisp
1051
+ calendar/contacts refresh runs POST-COMMIT — see
1052
+ :func:`_ingest_within_transaction` (Task 2.11).
1053
+
1054
+ Wave perf-T3 (F1): chunks + embeddings are PRECOMPUTED by the caller
1055
+ (outside the transaction) and passed in — this helper performs no Ollama
1056
+ calls, so the ``FOR UPDATE`` row lock it holds is never held across network
1057
+ I/O. Auto-summary enrichment is NOT run here; the caller invokes
1058
+ :func:`_enrich_post_ingest_hook` AFTER the commit.
1059
+
1060
+ Tag union semantics: existing curated tags on the row are preserved and
1061
+ merged with the incoming ``tags`` argument via
1062
+ ``normalize_tags([*existing, *incoming])``. Incoming tags are added;
1063
+ existing tags are never removed — ``brain tag`` is the only explicit
1064
+ tag-modification surface.
1065
+
1066
+ Summary-staleness rule: when ``body_changed`` is True this NULLs out
1067
+ ``summary`` / ``summary_model`` / ``summary_at`` in the same UPDATE that
1068
+ overwrites ``content_hash``, so a stale summary describing the old body can
1069
+ never linger if the post-commit enrich hook can't regenerate (``enrich=False``
1070
+ / ``enricher=None`` / content < min_tokens). When it can, the hook overwrites
1071
+ it. The caller passes ``body_changed`` on to the post-commit hook so its D11
1072
+ idempotency check knows the prior summary is stale. (Codex finding 1,
1073
+ 2026-05-11; Codex stop-gate finding, 2026-05-13.)
1074
+ """
1075
+ # F5 (Wave perf-T3): one locked SELECT reads BOTH the draft flag (for the
1076
+ # partial-window guard) AND the existing tags (for the union merge), where
1077
+ # the prior code issued two separate SELECTs against the same row. The
1078
+ # ``FOR UPDATE`` lock guards against concurrent tag-writes. ``body_changed``
1079
+ # is signalled via the kwarg, so we don't read content_hash here — the
1080
+ # caller computed it from the pre-UPDATE hash diff.
1081
+ existing_row = conn.execute(
1082
+ "SELECT draft, tags FROM documents WHERE id=%s FOR UPDATE",
1083
+ (document_id,),
1084
+ ).fetchone()
1085
+
1086
+ # Partial-window guard: if the incoming extraction is all-draft but the
1087
+ # existing row is published, the ingest window is a subset of the full
1088
+ # thread (e.g. ``brain ingest-gmail --since 7d`` where only a fresh
1089
+ # draft reply falls in the window but the thread's sent messages are
1090
+ # older). The draft-only extraction is NOT authoritative — refusing the
1091
+ # UPDATE preserves the published body, metadata, chunks, and links.
1092
+ # The legitimate auto-flip direction (TRUE→FALSE when a sent reply
1093
+ # arrives) is unaffected because this guard only triggers on draft=True.
1094
+ if draft and existing_row is not None and not existing_row[0]:
1095
+ return False
1096
+
1097
+ existing_tags: list[str] = (
1098
+ list(existing_row[1]) if existing_row and existing_row[1] else []
1099
+ )
1100
+
1101
+ # Union semantics: preserve existing curated tags; add incoming tags.
1102
+ # normalize_tags is idempotent (casefold + hyphenate) and preserves
1103
+ # first-seen insertion order, matching how apply_tags composes tags.
1104
+ merged_tags = normalize_tags([*existing_tags, *tags])
1105
+
1106
+ # Re-evaluate the source row — same ``(kind, external_id)`` as the
1107
+ # initial insert, so this is an ON CONFLICT no-op in practice. Kept here
1108
+ # so a future caller passing different ``source_metadata`` still resolves
1109
+ # the source row.
1110
+ source_id = _upsert_source(
1111
+ conn,
1112
+ kind=source_kind,
1113
+ external_id=source_external_id,
1114
+ metadata=source_metadata,
1115
+ )
1116
+
1117
+ # Pre-insert metadata derivation (krisp adds ``_participant_keys``).
1118
+ # No-op for gmail today, but kept symmetric with the INSERT path so
1119
+ # future per-source enrichments apply on UPDATE too.
1120
+ _apply_pre_insert_metadata(doc, source_kind=source_kind)
1121
+
1122
+ promoted = _promote_metadata_to_columns(doc.metadata)
1123
+ set_parts: list[str] = [
1124
+ "source_id=%s",
1125
+ "title=%s",
1126
+ "content=%s",
1127
+ "content_hash=%s",
1128
+ "content_type=%s",
1129
+ "tags=%s",
1130
+ "metadata=%s::jsonb",
1131
+ # Bump ingested_at on every content update so `brain status`'s
1132
+ # `last ingest` and the homepage recent-rail reflect actual ingest
1133
+ # activity, not just the original creation time. Default-NOW() on
1134
+ # INSERT means newly-created rows still get the right value.
1135
+ "ingested_at=NOW()",
1136
+ ]
1137
+ params: list[Any] = [
1138
+ source_id,
1139
+ doc.title,
1140
+ doc.content,
1141
+ content_hash,
1142
+ doc.content_type,
1143
+ merged_tags,
1144
+ json.dumps(doc.metadata),
1145
+ ]
1146
+
1147
+ # When the body changed, NULL out the stored summary so a stale summary
1148
+ # describing the OLD body can never be left on disk. The post-ingest
1149
+ # enrich hook regenerates the summary from NULL when an enricher is
1150
+ # available; when it isn't (enrich=False, enricher=None, or content
1151
+ # shorter than min_tokens) the hook returns early — but the row is
1152
+ # now correctly NULL-summary rather than carrying a misleading
1153
+ # description of the previous body. Complements master's body_changed
1154
+ # kwarg fix to _enrich_post_ingest_hook, which handles the
1155
+ # "regenerate when possible" path; this handles the "can't regenerate,
1156
+ # don't lie" path. (Codex stop-gate finding, 2026-05-13.)
1157
+ if body_changed:
1158
+ set_parts.extend(["summary=NULL", "summary_model=NULL", "summary_at=NULL"])
1159
+
1160
+ # Write the draft column directly. The entry-level partial-window
1161
+ # guard above already returned early if (existing=published,
1162
+ # incoming=draft-only), so reaching this point means either:
1163
+ # (a) draft=False — a normal sent-message update (auto-flip TRUE→FALSE
1164
+ # included), or
1165
+ # (b) draft=True AND existing was also draft=True (legitimate
1166
+ # all-draft refresh, e.g. a new draft was added to the thread).
1167
+ # In both cases the incoming ``draft`` value is authoritative.
1168
+ set_parts.append("draft=%s")
1169
+ params.append(draft)
1170
+ # Project the rebuilt thread's metadata onto the typed columns so they
1171
+ # stay in lockstep with the JSONB blob: a key that's present in the
1172
+ # new metadata writes the column; a key that's gone (e.g. an old
1173
+ # ``in_reply_to`` that no longer applies after a thread split) clears
1174
+ # the column to NULL.
1175
+ for column in _PROMOTED_COLUMNS:
1176
+ if column in promoted:
1177
+ set_parts.append(f"{column}=%s")
1178
+ params.append(promoted[column])
1179
+ else:
1180
+ set_parts.append(f"{column}=NULL")
1181
+ params.append(document_id)
1182
+ conn.execute(
1183
+ f"UPDATE documents SET {', '.join(set_parts)} WHERE id=%s",
1184
+ params,
1185
+ )
1186
+
1187
+ conn.execute("DELETE FROM chunks WHERE document_id=%s", (document_id,))
1188
+ # Use merged_tags (not the incoming tags arg) so chunks.tags_text reflects
1189
+ # the full union — existing curated tags stay searchable via the weighted-B
1190
+ # tsv field even after a re-ingest that passes an empty tags list.
1191
+ title_text, tags_text = _chunk_search_metadata(doc.title, merged_tags)
1192
+ _insert_chunks(
1193
+ conn,
1194
+ document_id=document_id,
1195
+ chunks=chunks,
1196
+ embeddings=embeddings,
1197
+ title_text=title_text,
1198
+ tags_text=tags_text,
1199
+ )
1200
+
1201
+ # In-transaction source hook: the Gmail directory upsert re-runs so headers
1202
+ # from the latest message in the rebuilt thread propagate; its rollback
1203
+ # contract is intentional. The slow Krisp calendar/contacts refresh runs
1204
+ # POST-COMMIT — see :func:`_ingest_within_transaction` (Task 2.11).
1205
+ _run_source_hooks(
1206
+ conn,
1207
+ source_kind=source_kind,
1208
+ doc=doc,
1209
+ document_id=document_id,
1210
+ )
1211
+ return True
1212
+
1213
+
1214
+ def _maybe_autotag_action_items(
1215
+ conn: psycopg.Connection,
1216
+ *,
1217
+ document_id: str,
1218
+ doc: ExtractedDoc,
1219
+ tags: list[str],
1220
+ ) -> None:
1221
+ """Apply the ``action-items`` tag to ``krisp_action_items`` docs.
1222
+
1223
+ Mechanical (no LLM), idempotent (the tag rewrite below collapses
1224
+ duplicates), and silent on non-action-items docs. Called once after
1225
+ the documents INSERT in :func:`_ingest_within_transaction`.
1226
+ """
1227
+ if doc.content_type != "krisp_action_items":
1228
+ return
1229
+ if "action-items" in tags:
1230
+ return # already present in the caller-supplied list
1231
+ # apply_tags re-normalizes; doing this in-transaction keeps the
1232
+ # documents row + tags atomic with the original ingest.
1233
+ apply_tags(conn, document_id, add=["action-items"])
1234
+
1235
+
1236
+ def _enrich_post_ingest_hook(
1237
+ conn: psycopg.Connection,
1238
+ *,
1239
+ document_id: str,
1240
+ doc: ExtractedDoc,
1241
+ enricher: OllamaEnricher | None,
1242
+ enrich: bool,
1243
+ min_tokens: int,
1244
+ content_hash: str,
1245
+ body_changed: bool = False,
1246
+ is_new_doc: bool = False,
1247
+ ) -> None:
1248
+ """Generate + persist a 2-3 sentence summary on ``documents.summary``.
1249
+
1250
+ On the ingest hot path (Wave perf-T3 / F1) this runs AFTER the document
1251
+ transaction has committed, in autocommit mode, as its own short ``UPDATE``
1252
+ — a failure here leaves the committed row with ``summary IS NULL`` and never
1253
+ rolls back the ingest. On the ``update_document`` (``brain edit``) path and
1254
+ in direct unit tests it still runs inside the caller's open transaction.
1255
+ Skips silently (with a debug log) when any of the wave-plan §3.a rules
1256
+ apply:
1257
+
1258
+ 1. ``enrich=False`` — caller passed ``--no-enrich`` or a library caller
1259
+ deliberately disabled it.
1260
+ 2. ``enricher is None`` — caller didn't wire one up (legit for tests
1261
+ and internal pipelines that don't want LLM round-trips).
1262
+ 3. Content shorter than ``min_tokens`` tokens — the title alone is
1263
+ already a fine summary; no point spending an LLM round-trip.
1264
+ 4. ``documents.summary IS NOT NULL`` AND ``content_hash`` unchanged
1265
+ AND ``body_changed`` is False — idempotency: same body, same model
1266
+ → reuse the prior summary (D11).
1267
+ 5. ``OllamaUnavailable`` — Ollama is down / 5xx; logged at WARN. The
1268
+ ingest still commits; ``brain enrich --backfill`` picks the row up
1269
+ later.
1270
+ 6. :class:`EnrichmentError` (model returned malformed JSON twice in a
1271
+ row) — logged at WARN; row stays unenriched.
1272
+
1273
+ The ``body_changed`` kwarg is the Codex finding 1 (2026-05-11) fix.
1274
+ Update-path callers (:func:`_update_thread_doc_in_place`,
1275
+ :func:`update_document`) UPDATE ``documents.content_hash`` to the new
1276
+ value BEFORE invoking this hook. The hook then SELECTs the row, reads
1277
+ the just-overwritten hash, and the ``existing_hash == content_hash``
1278
+ comparison is meaningless — both sides equal the new hash. The
1279
+ pre-fix behavior fell through the D11 skip and left the prior
1280
+ (stale) summary in place even when the body had changed, which
1281
+ Q2-SUMMARY-WIKI then rendered above the article body as a wiki
1282
+ lede — user-visible drift. The fix: callers that know the body
1283
+ changed (or might have) pass ``body_changed=True``; the D11 skip
1284
+ only fires when ``body_changed`` is False AND the other conditions
1285
+ hold. The INSERT call site keeps the default ``False`` because the
1286
+ row is freshly INSERTed with ``summary IS NULL`` — the first clause
1287
+ of the D11 guard naturally short-circuits there.
1288
+ """
1289
+ if not enrich:
1290
+ return
1291
+ if enricher is None:
1292
+ _logger.debug(
1293
+ "enrich hook: no enricher supplied for %s; skipping", document_id
1294
+ )
1295
+ return
1296
+ if enricher.count_tokens(doc.content) < min_tokens:
1297
+ return
1298
+ # F4 (Wave perf-T3): on the new-doc path the row was just INSERTed with
1299
+ # ``summary IS NULL`` / ``content_hash == content_hash`` / ``summary_model
1300
+ # IS NULL``, so the D11 idempotency SELECT can only ever read those known
1301
+ # values and never skips — drop the round-trip and proceed straight to
1302
+ # summarization. The update path (``is_new_doc=False``) keeps the SELECT so
1303
+ # D11 can reuse a still-valid summary.
1304
+ if not is_new_doc:
1305
+ row = conn.execute(
1306
+ "SELECT summary, content_hash, summary_model FROM documents WHERE id=%s",
1307
+ (document_id,),
1308
+ ).fetchone()
1309
+ if row is None:
1310
+ return # defensive — the row was just INSERTed
1311
+ existing_summary, existing_hash, existing_model = row
1312
+ # Idempotency D11: reuse prior summary ONLY when the body AND model both
1313
+ # match the current enricher AND the caller hasn't told us the body
1314
+ # just changed. After a ``BRAIN_ENRICH_MODEL`` upgrade (e.g.,
1315
+ # llama3.1:8b → llama3.2:8b), the existing summary is stale relative
1316
+ # to the new model — re-enrich so improvements propagate.
1317
+ if (
1318
+ existing_summary is not None
1319
+ and not body_changed
1320
+ and existing_hash == content_hash
1321
+ and existing_model == enricher.model
1322
+ ):
1323
+ return
1324
+ try:
1325
+ result = enricher.summarize(doc.title, doc.content)
1326
+ except OllamaUnavailable as exc:
1327
+ _logger.warning(
1328
+ "auto-summary skipped for %s: Ollama unavailable (%s); "
1329
+ "run `brain enrich --backfill` later",
1330
+ document_id,
1331
+ exc,
1332
+ )
1333
+ return
1334
+ except EnrichmentError as exc:
1335
+ _logger.warning(
1336
+ "auto-summary failed for %s: %s; row marked unenriched",
1337
+ document_id,
1338
+ exc,
1339
+ )
1340
+ return
1341
+ conn.execute(
1342
+ "UPDATE documents SET summary=%s, summary_model=%s, summary_at=NOW() "
1343
+ "WHERE id=%s",
1344
+ (result.summary, result.model, document_id),
1345
+ )
1346
+
1347
+
1348
+ def _apply_pre_insert_metadata(doc: ExtractedDoc, *, source_kind: str) -> None:
1349
+ """Mutate ``doc.metadata`` in place to add derived fields by source.
1350
+
1351
+ Currently:
1352
+ - ``krisp`` → ``_participant_keys`` (sorted list of normalized speaker
1353
+ keys parsed from the transcript body). Always present after this call,
1354
+ even if the body has no speaker labels (empty list signals "this doc
1355
+ was processed by the linker pre-stage" to downstream code).
1356
+ """
1357
+ if source_kind == "krisp":
1358
+ doc.metadata["_participant_keys"] = sorted(extract_krisp_speakers(doc.content))
1359
+
1360
+
1361
+ def _gmail_post_ingest_hook(
1362
+ conn: psycopg.Connection, doc: ExtractedDoc, document_id: str
1363
+ ) -> None:
1364
+ """Upsert (display_name, email) pairs from the Gmail From/To headers.
1365
+
1366
+ Runs inside the outer ``conn.transaction()`` opened by
1367
+ :func:`ingest_document` so a directory-write failure rolls the
1368
+ document back too. ``document_id`` is unused today but kept in the
1369
+ signature for symmetry with future hooks that may need it.
1370
+ """
1371
+ del document_id # symmetry with other source hooks
1372
+ store = DirectoryStore(conn)
1373
+ for display_name, email in extract_gmail_addresses(doc.metadata):
1374
+ store.upsert_pair(
1375
+ display_name=display_name,
1376
+ email=email,
1377
+ source="gmail",
1378
+ )
1379
+
1380
+
1381
+ def _krisp_post_ingest_hook(
1382
+ conn: psycopg.Connection,
1383
+ doc: ExtractedDoc,
1384
+ document_id: str,
1385
+ runner: GwsRunner | None,
1386
+ ) -> None:
1387
+ """Trigger an incremental Calendar refresh + a stale-only Contacts refresh.
1388
+
1389
+ Both refreshes degrade soft: ``refresh_calendar`` / ``refresh_contacts``
1390
+ catch runner failures internally and return 0 so a Krisp ingest never
1391
+ fails on a transient gws hiccup.
1392
+
1393
+ Calendar window:
1394
+ - First run (no ``directory_refresh_state`` row for ``calendar``) →
1395
+ since = ``YYYY-01-01T00:00:00+00:00`` (current-year start, UTC).
1396
+ - Subsequent runs → since = the stored ``last_refreshed_at``.
1397
+ - until = ``datetime.now(tz=UTC)`` (always).
1398
+
1399
+ Contacts cadence:
1400
+ - Runs only if no state row exists OR ``last_refreshed_at`` is older
1401
+ than 24 hours. Prevents re-fetching the People API on every ingest.
1402
+ """
1403
+ del doc, document_id # unused — krisp metadata mutation happens pre-insert
1404
+ if runner is None:
1405
+ _logger.warning(
1406
+ "krisp post-ingest: no gws_runner provided; "
1407
+ "skipping calendar/contacts refresh"
1408
+ )
1409
+ return
1410
+
1411
+ now = datetime.now(tz=UTC)
1412
+
1413
+ cal_row = conn.execute(
1414
+ "SELECT last_refreshed_at FROM directory_refresh_state "
1415
+ "WHERE source = 'calendar'"
1416
+ ).fetchone()
1417
+ if cal_row is not None and cal_row[0] is not None:
1418
+ since = cal_row[0]
1419
+ else:
1420
+ since = datetime(now.year, 1, 1, tzinfo=UTC)
1421
+ refresh_calendar(conn, since=since, until=now, runner=runner)
1422
+
1423
+ contacts_row = conn.execute(
1424
+ "SELECT last_refreshed_at FROM directory_refresh_state "
1425
+ "WHERE source = 'contacts'"
1426
+ ).fetchone()
1427
+ contacts_stale = (
1428
+ contacts_row is None
1429
+ or contacts_row[0] is None
1430
+ or contacts_row[0] < now - _CONTACTS_REFRESH_INTERVAL
1431
+ )
1432
+ if contacts_stale:
1433
+ refresh_contacts(conn, runner=runner)
1434
+
1435
+
1436
+ def _run_source_hooks(
1437
+ conn: psycopg.Connection,
1438
+ *,
1439
+ source_kind: str,
1440
+ doc: ExtractedDoc,
1441
+ document_id: str,
1442
+ ) -> None:
1443
+ """Dispatch the IN-TRANSACTION source-specific post-ingest hooks.
1444
+
1445
+ Only Gmail's directory upsert runs here — it is a fast local DB write
1446
+ (no network / subprocess) with a deliberate atomic-rollback contract: a
1447
+ directory-write failure rolls the whole document back
1448
+ (``test_directory_upsert_runs_in_same_transaction``). Keeping it in the
1449
+ write transaction is a preserved guarantee.
1450
+
1451
+ Krisp's calendar / contacts refresh does NOT run here — it shells out to
1452
+ ``gws`` (multi-second subprocess / network calls) and would otherwise hold
1453
+ the write transaction open, so it runs POST-COMMIT via
1454
+ :func:`_run_post_commit_source_hooks` (Task 2.11).
1455
+
1456
+ Sources without an in-transaction hook are a no-op. A new fast/atomic hook
1457
+ adds a branch here; a new slow / soft-degrading hook adds a branch to
1458
+ :func:`_run_post_commit_source_hooks` instead (Open/Closed).
1459
+ """
1460
+ if source_kind == "gmail":
1461
+ _gmail_post_ingest_hook(conn, doc, document_id)
1462
+
1463
+
1464
+ def _run_post_commit_source_hooks(
1465
+ conn: psycopg.Connection,
1466
+ *,
1467
+ source_kind: str,
1468
+ doc: ExtractedDoc,
1469
+ document_id: str,
1470
+ gws_runner: GwsRunner | None,
1471
+ ) -> None:
1472
+ """Dispatch the POST-COMMIT source-specific post-ingest hooks (Task 2.11).
1473
+
1474
+ Only Krisp's calendar + contacts refresh runs here. It shells out to the
1475
+ ``gws`` CLI (multi-second subprocess / network calls) and soft-degrades —
1476
+ ``refresh_calendar`` / ``refresh_contacts`` catch runner failures and
1477
+ return 0 — so running it AFTER the document transaction commits holds no
1478
+ write lock across the slow refresh and loses no rollback semantics.
1479
+
1480
+ Gmail's directory upsert runs IN-TRANSACTION via :func:`_run_source_hooks`
1481
+ because it has an atomic-rollback contract; keeping it there is a
1482
+ deliberately preserved guarantee, NOT an oversight.
1483
+
1484
+ **Post-commit failure containment.** This runs AFTER the document
1485
+ transaction has already committed (Task 2.11), so a failure here must never
1486
+ surface as an ingest failure — the row is durably saved. The refresh helpers
1487
+ already soft-degrade on ``gws``/runner failures, but the hook still issues its
1488
+ own ``directory_refresh_state`` SELECTs, whose raw ``psycopg.Error`` (e.g. a
1489
+ connection dropped after commit) would otherwise propagate out of
1490
+ :func:`ingest_document`. We catch it, log a WARN, and return — mirroring the
1491
+ :func:`_enrich_post_ingest_hook` guard (the calendar/contacts refresh is
1492
+ simply skipped; a later ingest re-triggers it).
1493
+ """
1494
+ try:
1495
+ if source_kind == "krisp":
1496
+ _krisp_post_ingest_hook(conn, doc, document_id, gws_runner)
1497
+ except psycopg.Error as exc:
1498
+ _logger.warning(
1499
+ "post-commit source hook for %s failed (%s); ingest already "
1500
+ "committed — calendar/contacts refresh skipped",
1501
+ source_kind,
1502
+ exc,
1503
+ )
1504
+
1505
+
1506
+ def apply_tags(
1507
+ conn: psycopg.Connection,
1508
+ document_id: str,
1509
+ *,
1510
+ add: list[str] | None = None,
1511
+ remove: list[str] | None = None,
1512
+ ) -> list[str]:
1513
+ """Add and/or remove tags on a document; return the resulting tag list.
1514
+
1515
+ ``add`` is unioned with the existing tags (idempotent — re-adding an
1516
+ existing tag is a no-op); ``remove`` strips any matching tags. Operations
1517
+ run in a single transaction. Caller is responsible for resolving any
1518
+ UUID prefix to a full ``document_id`` before calling.
1519
+
1520
+ Inputs are passed through :func:`brain.tags.normalize_tags` before the
1521
+ DB write — this is the single ongoing-enforcement boundary that keeps
1522
+ ``brain tag <id> +COMPANY_REDACTED`` storing ``company-ko`` and lets a remove of
1523
+ ``COMPANY_REDACTED`` match a row that's currently stored as ``company-ko``.
1524
+ """
1525
+ # Deferred import: ``brain.queries`` imports the ``Embedder`` Protocol
1526
+ # from this module, so a top-level ``from ..queries import ...`` would
1527
+ # cycle at package-load time. The function-local import resolves cleanly
1528
+ # because both modules have finished initializing by the time
1529
+ # ``apply_tags`` is first called.
1530
+ from ..queries import sync_chunk_search_metadata
1531
+
1532
+ add = normalize_tags(add or [])
1533
+ remove = normalize_tags(remove or [])
1534
+ with conn.transaction():
1535
+ if add:
1536
+ conn.execute(
1537
+ "UPDATE documents SET tags = ARRAY(SELECT DISTINCT unnest(tags || %s::text[])) "
1538
+ "WHERE id = %s",
1539
+ (add, document_id),
1540
+ )
1541
+ if remove:
1542
+ conn.execute(
1543
+ "UPDATE documents SET tags = ARRAY(SELECT t FROM unnest(tags) AS t "
1544
+ "WHERE t <> ALL(%s::text[])) WHERE id = %s",
1545
+ (remove, document_id),
1546
+ )
1547
+ # Migration 009 denormalizes documents.tags onto chunks.tags_text so
1548
+ # the weighted tsv reflects tag changes. Run inside the same
1549
+ # transaction that wrote the new tags so a rollback (e.g. SELECT
1550
+ # below failing) keeps chunks and parent in lockstep. The IS
1551
+ # DISTINCT FROM guards in the helper make this a no-op when add /
1552
+ # remove produced no net change.
1553
+ if add or remove:
1554
+ sync_chunk_search_metadata(conn, document_id)
1555
+ row = conn.execute(
1556
+ "SELECT tags FROM documents WHERE id=%s", (document_id,)
1557
+ ).fetchone()
1558
+ if row is None:
1559
+ raise ValueError(f"document not found: {document_id}")
1560
+ return list(row[0] or [])
1561
+
1562
+
1563
+ _MIRROR_FRONTMATTER_FIELDS = frozenset(
1564
+ {"title", "tags", "metadata", "content_type", "draft", "summary"}
1565
+ )
1566
+
1567
+
1568
+ def update_document(
1569
+ conn: psycopg.Connection,
1570
+ *,
1571
+ document_id: str,
1572
+ embedder: Embedder | None = None,
1573
+ new_title: str | None = None,
1574
+ new_content_type: str | None = None,
1575
+ new_content: str | None = None,
1576
+ metadata_patch: dict[str, Any] | None = None,
1577
+ replace_metadata: bool = False,
1578
+ new_tags: list[str] | None = None,
1579
+ new_draft: bool | None = None,
1580
+ vault_root: Path | None = None,
1581
+ enricher: OllamaEnricher | None = None,
1582
+ enrich: bool = True,
1583
+ enrich_min_tokens: int = 50,
1584
+ graph_syncer: GraphSyncer | None = None,
1585
+ ) -> UpdateResult:
1586
+ """Update one document in place.
1587
+
1588
+ Body changes (``new_content``) re-chunk + re-embed atomically: the prior
1589
+ chunks are deleted and new ones are inserted in the same transaction.
1590
+ Metadata defaults to a shallow merge — top-level keys overwrite, nested
1591
+ objects are not deep-merged. Set ``replace_metadata=True`` to swap the
1592
+ blob entirely.
1593
+
1594
+ ``new_draft`` flips the top-level ``documents.draft`` boolean column
1595
+ introduced by migration 007. Unlike metadata-promoted columns this lives
1596
+ directly on ``documents``; passing ``True`` / ``False`` writes the new
1597
+ value, ``None`` leaves it alone. The wiki build (Quartz contentIndex
1598
+ emitter) hides ``draft=true`` docs from the explorer/graph/search;
1599
+ ``brain search`` / ``brain list`` still surface them so the user can
1600
+ re-publish.
1601
+
1602
+ Raises :class:`ValueError` if ``new_content`` is empty/whitespace-only or
1603
+ if its SHA-256 collides with another document. ``embedder`` is required
1604
+ when ``new_content`` is provided. Empty/no-op edits are not an error and
1605
+ return an :class:`UpdateResult` with ``fields_changed=[]``.
1606
+
1607
+ Vault mirror: when ``vault_root`` is supplied AND the edit actually
1608
+ changed the body or any frontmatter-bearing field
1609
+ (``title`` / ``tags`` / ``metadata`` / ``content_type`` / ``draft``),
1610
+ the mirror file under ``vault_root`` is regenerated via
1611
+ :func:`brain.vault.export.regenerate_vault_file`. Vault-tier rows
1612
+ (``kind='vault'``) are skipped via a DB pre-check — those files are
1613
+ file-source-of-truth and ``vault sync`` reconciles back to the DB.
1614
+ A filesystem error (``OSError``) on the mirror write logs a WARNING
1615
+ and does NOT roll back the DB update — drift is recoverable via
1616
+ ``brain vault export --force``. Named ``vault_root`` to distinguish
1617
+ from the per-document ``documents.vault_path`` relative path.
1618
+
1619
+ Wave G1-c graph sync: when ``graph_syncer`` is supplied AND any field
1620
+ changed, the document's people-aspect graph is reconciled post-commit via
1621
+ :meth:`brain.graph_rag.sync.GraphSyncer.reconcile` (reusing ``conn``,
1622
+ OUTSIDE the transaction). Best-effort / never-raises; a ``None`` syncer or a
1623
+ no-op edit skips it.
1624
+ """
1625
+ with conn.transaction():
1626
+ row = conn.execute(
1627
+ "SELECT title, content, content_type, metadata, tags, kind, draft "
1628
+ "FROM documents WHERE id=%s",
1629
+ (document_id,),
1630
+ ).fetchone()
1631
+ if row is None:
1632
+ raise ValueError(f"document not found: {document_id}")
1633
+ (
1634
+ cur_title,
1635
+ cur_content,
1636
+ cur_type,
1637
+ cur_meta,
1638
+ cur_tags,
1639
+ cur_kind,
1640
+ cur_draft,
1641
+ ) = row
1642
+ cur_meta = dict(cur_meta or {})
1643
+ cur_tags = list(cur_tags or [])
1644
+
1645
+ fields_changed: list[str] = []
1646
+ sets: list[str] = []
1647
+ params: list[Any] = []
1648
+
1649
+ rechunked = False
1650
+ new_hash: str | None = None
1651
+ if new_content is not None:
1652
+ if embedder is None:
1653
+ raise ValueError("embedder is required when new_content is provided")
1654
+ stripped = new_content.strip()
1655
+ if not stripped:
1656
+ raise ValueError("content is empty")
1657
+ if new_content != cur_content:
1658
+ new_hash = _content_hash(new_content)
1659
+ clash = conn.execute(
1660
+ "SELECT id FROM documents WHERE content_hash=%s AND id<>%s",
1661
+ (new_hash, document_id),
1662
+ ).fetchone()
1663
+ if clash:
1664
+ raise ValueError(
1665
+ f"content collides with existing document {clash[0]}"
1666
+ )
1667
+ rechunked = True
1668
+
1669
+ if new_title is not None and new_title != cur_title:
1670
+ sets.append("title=%s")
1671
+ params.append(new_title)
1672
+ fields_changed.append("title")
1673
+
1674
+ if new_content_type is not None and new_content_type != cur_type:
1675
+ sets.append("content_type=%s")
1676
+ params.append(new_content_type)
1677
+ fields_changed.append("content_type")
1678
+
1679
+ if metadata_patch is not None:
1680
+ new_meta = (
1681
+ metadata_patch if replace_metadata else {**cur_meta, **metadata_patch}
1682
+ )
1683
+ if new_meta != cur_meta:
1684
+ sets.append("metadata=%s::jsonb")
1685
+ params.append(json.dumps(new_meta))
1686
+ fields_changed.append("metadata")
1687
+ # Re-project the new metadata blob onto typed columns, and
1688
+ # diff against what we'd project from the prior blob. This
1689
+ # keeps the typed columns in lockstep with metadata: a
1690
+ # patch that adds ``thread_id`` populates the column; a
1691
+ # replace_metadata that drops the key NULLs the column.
1692
+ old_promoted = _promote_metadata_to_columns(cur_meta)
1693
+ new_promoted = _promote_metadata_to_columns(new_meta)
1694
+ for column in _PROMOTED_COLUMNS:
1695
+ if column in new_promoted:
1696
+ if new_promoted[column] != old_promoted.get(column):
1697
+ sets.append(f"{column}=%s")
1698
+ params.append(new_promoted[column])
1699
+ elif column in old_promoted:
1700
+ # replace_metadata dropped the source key — clear
1701
+ # the typed column so it no longer disagrees with
1702
+ # the JSONB blob.
1703
+ sets.append(f"{column}=NULL")
1704
+
1705
+ if new_tags is not None and sorted(new_tags) != sorted(cur_tags):
1706
+ sets.append("tags=%s")
1707
+ params.append(list(new_tags))
1708
+ fields_changed.append("tags")
1709
+
1710
+ if new_draft is not None and bool(new_draft) != bool(cur_draft):
1711
+ sets.append("draft=%s")
1712
+ params.append(bool(new_draft))
1713
+ fields_changed.append("draft")
1714
+
1715
+ if rechunked:
1716
+ assert new_hash is not None # set above when rechunked is True
1717
+ assert embedder is not None # checked above
1718
+ sets.append("content=%s")
1719
+ params.append(new_content)
1720
+ sets.append("content_hash=%s")
1721
+ params.append(new_hash)
1722
+ fields_changed.append("content")
1723
+ # Same rationale as ``_update_doc_in_place``: NULL the stored
1724
+ # summary in the SAME UPDATE that overwrites content_hash so a
1725
+ # stale summary describing the old body can never linger when
1726
+ # the post-update enrich hook can't regenerate (``enrich=False``,
1727
+ # ``enricher=None``, or content < min_tokens). The hook below
1728
+ # regenerates from NULL when an enricher is available; otherwise
1729
+ # the row is correctly NULL-summary rather than a misleading
1730
+ # description of the previous content. (Codex stop-gate finding,
1731
+ # 2026-05-13.)
1732
+ sets.append("summary=NULL")
1733
+ sets.append("summary_model=NULL")
1734
+ sets.append("summary_at=NULL")
1735
+
1736
+ conn.execute(
1737
+ "DELETE FROM chunks WHERE document_id=%s", (document_id,)
1738
+ )
1739
+ assert new_content is not None # gated by the empty-check above
1740
+ chunks = chunk_text(new_content, count_tokens=embedder.count_tokens)
1741
+ if chunks:
1742
+ embeddings = _embed_chunks(embedder, [c.content for c in chunks])
1743
+ # Project the post-update title/tags onto the new chunks so
1744
+ # the weighted tsv reflects the user's new edits, not the
1745
+ # pre-edit state. Falls back to current values when the
1746
+ # caller didn't pass an override; the surrounding ``UPDATE
1747
+ # documents`` writes the same values back to the documents
1748
+ # row, so the chunks and parent stay in lockstep.
1749
+ final_title = new_title if new_title is not None else cur_title
1750
+ final_tags = list(new_tags) if new_tags is not None else cur_tags
1751
+ title_text, tags_text = _chunk_search_metadata(final_title, final_tags)
1752
+ # F2 (Wave perf-T3): single executemany over the rebuilt chunks
1753
+ # instead of a per-chunk round-trip. Still inside this txn — the
1754
+ # ``brain edit`` path is interactive/single-doc, not the batch
1755
+ # ingest hot path, so its embed/enrich stay transactional.
1756
+ _insert_chunks(
1757
+ conn,
1758
+ document_id=document_id,
1759
+ chunks=chunks,
1760
+ embeddings=embeddings,
1761
+ title_text=title_text,
1762
+ tags_text=tags_text,
1763
+ )
1764
+
1765
+ if sets:
1766
+ # Same rationale as update_document: bump ingested_at so
1767
+ # `brain status`'s last-ingest stat and the vault-export
1768
+ # `updated:` frontmatter field reflect actual edits.
1769
+ sets.append("ingested_at=NOW()")
1770
+ params.append(document_id)
1771
+ conn.execute(
1772
+ f"UPDATE documents SET {', '.join(sets)} WHERE id=%s",
1773
+ params,
1774
+ )
1775
+
1776
+ # Migration 009 denormalizes documents.title / documents.tags onto
1777
+ # chunks.title_text / chunks.tags_text. After ``rechunked`` the
1778
+ # freshly-inserted chunks already carry the post-update values, so
1779
+ # the helper's IS DISTINCT FROM guards turn this into a no-op for
1780
+ # those rows. For the no-rechunk path (title-only or tags-only
1781
+ # edit), this is the call that propagates the change to every
1782
+ # existing chunk so the weighted tsv stays consistent.
1783
+ if "title" in fields_changed or "tags" in fields_changed:
1784
+ from ..queries import sync_chunk_search_metadata
1785
+ sync_chunk_search_metadata(conn, document_id)
1786
+
1787
+ # Wave Q1-D — re-enrich on body change. The ``rechunked`` gate
1788
+ # above only fires when ``new_content != cur_content``, so reaching
1789
+ # this point means the body provably changed — pass
1790
+ # ``body_changed=True`` so the D11 idempotency check inside
1791
+ # ``_enrich_post_ingest_hook`` knows the existing summary is now
1792
+ # stale relative to the just-written body (Codex finding 1 fix —
1793
+ # without this kwarg the hook reads back the just-updated row
1794
+ # whose ``content_hash`` already matches ``new_hash``, the
1795
+ # ``existing_hash == content_hash`` check trivially passes, and
1796
+ # the hook short-circuits leaving the prior body's summary
1797
+ # rendered in the Q2-SUMMARY-WIKI lede above the new body).
1798
+ # Build an ExtractedDoc-shaped object lazily — we only need
1799
+ # ``title`` and ``content`` (the parts the enricher reads).
1800
+ # Re-uses the post-update title/content so an in-flight title
1801
+ # edit sees the new one.
1802
+ if rechunked:
1803
+ final_title = new_title if new_title is not None else cur_title
1804
+ assert new_content is not None # rechunked implies new_content set
1805
+ new_doc = ExtractedDoc(
1806
+ title=final_title,
1807
+ content=new_content,
1808
+ content_type=cur_type,
1809
+ source_path=None,
1810
+ metadata={},
1811
+ )
1812
+ assert new_hash is not None # rechunked implies new_hash set
1813
+ _enrich_post_ingest_hook(
1814
+ conn,
1815
+ document_id=document_id,
1816
+ doc=new_doc,
1817
+ enricher=enricher,
1818
+ enrich=enrich,
1819
+ min_tokens=enrich_min_tokens,
1820
+ content_hash=new_hash,
1821
+ body_changed=True,
1822
+ )
1823
+
1824
+ # Mirror writes happen OUTSIDE the transaction so a filesystem error
1825
+ # cannot roll back the DB update. Triggered when the body was rechunked
1826
+ # OR any frontmatter-derived field changed (the mirror's frontmatter is
1827
+ # built from documents.title/tags/metadata/content_type — a metadata-only
1828
+ # edit must still propagate). Vault-tier rows are skipped via the DB
1829
+ # ``cur_kind`` pre-check rather than by catching ``regenerate_vault_file``'s
1830
+ # ValueError — pre-checking ``kind`` from the DB is more robust than
1831
+ # catching a ``ValueError`` and string-matching its message, since the
1832
+ # upstream message can be rephrased without notice.
1833
+ needs_mirror = rechunked or any(
1834
+ f in _MIRROR_FRONTMATTER_FIELDS for f in fields_changed
1835
+ )
1836
+ if vault_root is not None and cur_kind != "vault" and needs_mirror:
1837
+ try:
1838
+ # ``force=True`` because we've already gated on
1839
+ # ``rechunked or any(... in _MIRROR_FRONTMATTER_FIELDS ...)`` —
1840
+ # a frontmatter-only edit (tags / metadata / content_type) leaves
1841
+ # the body unchanged, so the body-hash skip in ``_write_doc_file``
1842
+ # would silently drop the rewrite and leave stale frontmatter on
1843
+ # disk. The full-corpus ``export_vault`` keeps the default
1844
+ # ``force=False`` so re-runs of that path stay cheap.
1845
+ regenerate_vault_file(
1846
+ conn, document_id, vault_path=vault_root, force=True
1847
+ )
1848
+ except OSError as exc:
1849
+ # Only OSError is reachable: ``no document with id`` is impossible
1850
+ # (we just SELECTed it), and ``kind='vault'`` is gated above.
1851
+ _logger.warning(
1852
+ "vault mirror write failed for document %s: %s; "
1853
+ "DB update succeeded — recover via `brain vault export`",
1854
+ document_id,
1855
+ exc,
1856
+ )
1857
+
1858
+ # Wave G1-c — people-aspect graph sync after an in-place edit. Runs
1859
+ # post-commit (reusing ``conn``) whenever any field actually changed; a
1860
+ # change to the participants metadata flips the reconcile watermark and
1861
+ # re-indexes, while a person-irrelevant edit (e.g. a ``draft`` toggle) is a
1862
+ # cheap watermark-skip. Best-effort / never-raises — see
1863
+ # :meth:`brain.graph_rag.sync.GraphSyncer.reconcile`. A ``None`` syncer or a
1864
+ # no-op edit (empty ``fields_changed``) skips graph sync entirely.
1865
+ if graph_syncer is not None and fields_changed:
1866
+ graph_syncer.reconcile(conn, document_id)
1867
+
1868
+ return UpdateResult(
1869
+ document_id=document_id,
1870
+ fields_changed=fields_changed,
1871
+ rechunked=rechunked,
1872
+ )
1873
+
1874
+
1875
+ _EXTRACTORS = {
1876
+ ".txt": "text",
1877
+ ".md": "markdown",
1878
+ ".markdown": "markdown",
1879
+ ".pdf": "pdf",
1880
+ ".docx": "docx",
1881
+ }
1882
+
1883
+
1884
+ def extract_path(path: Path) -> ExtractedDoc:
1885
+ """Dispatch to the correct extractor based on file extension.
1886
+
1887
+ Raises ``ValueError`` for unsupported extensions or malformed input files;
1888
+ ``OSError`` for unreadable files. Backend-specific parser errors (e.g.
1889
+ :class:`pypdf.errors.PyPdfError`) are wrapped as ``ValueError`` so callers
1890
+ only need to handle a narrow set of exception types.
1891
+ """
1892
+ ext = Path(path).suffix.lower()
1893
+ name = _EXTRACTORS.get(ext)
1894
+ if name is None:
1895
+ raise ValueError(f"unsupported file type: {ext}")
1896
+ try:
1897
+ if name == "text":
1898
+ from .text import extract_text
1899
+ return extract_text(path)
1900
+ if name == "markdown":
1901
+ from .markdown import extract_markdown
1902
+ return extract_markdown(path)
1903
+ if name == "pdf":
1904
+ from pypdf.errors import PyPdfError
1905
+
1906
+ from .pdf import extract_pdf
1907
+ try:
1908
+ return extract_pdf(path)
1909
+ except PyPdfError as e:
1910
+ raise ValueError(f"malformed PDF: {e}") from e
1911
+ if name == "docx":
1912
+ from docx.opc.exceptions import PackageNotFoundError
1913
+
1914
+ from .docx import extract_docx
1915
+ try:
1916
+ return extract_docx(path)
1917
+ except PackageNotFoundError as e:
1918
+ raise ValueError(f"malformed DOCX: {e}") from e
1919
+ except UnicodeDecodeError as e: # pragma: no cover - extractors use errors="replace"
1920
+ raise ValueError(f"could not decode file as UTF-8: {e}") from e
1921
+ raise AssertionError("unreachable") # pragma: no cover
1922
+
1923
+
1924
+ def supported_extensions() -> list[str]:
1925
+ """Return the list of file extensions that :func:`extract_path` can handle."""
1926
+ return list(_EXTRACTORS.keys())