secondbrain-py 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. brain/__init__.py +0 -0
  2. brain/__main__.py +18 -0
  3. brain/_capture_command.py +445 -0
  4. brain/_compose.py +52 -0
  5. brain/activity.py +206 -0
  6. brain/ask.py +631 -0
  7. brain/audio.py +591 -0
  8. brain/backfill/__init__.py +12 -0
  9. brain/backfill/search_extras.py +141 -0
  10. brain/backfill/source_rows.py +101 -0
  11. brain/bin/__init__.py +1 -0
  12. brain/bin/_launcher.py +107 -0
  13. brain/bin/down.py +8 -0
  14. brain/bin/launchd.py +268 -0
  15. brain/bin/monitor.py +570 -0
  16. brain/bin/rebuild.py +8 -0
  17. brain/bin/status.py +8 -0
  18. brain/bin/up.py +8 -0
  19. brain/brief.py +272 -0
  20. brain/capture.py +49 -0
  21. brain/chat.py +293 -0
  22. brain/cli.py +9760 -0
  23. brain/cli_claude.py +81 -0
  24. brain/cli_connect.py +285 -0
  25. brain/cli_demo.py +266 -0
  26. brain/config.py +1949 -0
  27. brain/connect.py +925 -0
  28. brain/db.py +540 -0
  29. brain/demo/__init__.py +452 -0
  30. brain/demo/corpus/manifest.json +403 -0
  31. brain/demo/embedder.py +74 -0
  32. brain/durations.py +84 -0
  33. brain/edit_session.py +156 -0
  34. brain/editor.py +67 -0
  35. brain/elicit/__init__.py +16 -0
  36. brain/elicit/detectors.py +250 -0
  37. brain/elicit/drafter.py +70 -0
  38. brain/elicit/queue.py +220 -0
  39. brain/elicit/schema.py +48 -0
  40. brain/elicit/session.py +445 -0
  41. brain/embedding_targets.py +54 -0
  42. brain/embeddings.py +424 -0
  43. brain/enrichment.py +808 -0
  44. brain/errors.py +357 -0
  45. brain/eval/__init__.py +129 -0
  46. brain/eval/answer_eval.py +281 -0
  47. brain/eval/baseline.py +265 -0
  48. brain/eval/concept_extraction.py +378 -0
  49. brain/eval/corpus.py +152 -0
  50. brain/eval/errors.py +19 -0
  51. brain/eval/graph_baseline.py +226 -0
  52. brain/eval/graph_retrieval.py +202 -0
  53. brain/eval/graph_runner.py +319 -0
  54. brain/eval/metrics.py +101 -0
  55. brain/eval/runner.py +223 -0
  56. brain/format.py +783 -0
  57. brain/gaps.py +390 -0
  58. brain/graph_rag/__init__.py +94 -0
  59. brain/graph_rag/_retrieval_common.py +113 -0
  60. brain/graph_rag/aggregates.py +303 -0
  61. brain/graph_rag/aliases/__init__.py +583 -0
  62. brain/graph_rag/backends/__init__.py +10 -0
  63. brain/graph_rag/backends/_age_helpers.py +473 -0
  64. brain/graph_rag/backends/age.py +782 -0
  65. brain/graph_rag/backends/base.py +272 -0
  66. brain/graph_rag/build.py +344 -0
  67. brain/graph_rag/communities.py +644 -0
  68. brain/graph_rag/communities_summary.py +437 -0
  69. brain/graph_rag/concepts.py +202 -0
  70. brain/graph_rag/cooccur.py +193 -0
  71. brain/graph_rag/cross_type.py +312 -0
  72. brain/graph_rag/extract.py +885 -0
  73. brain/graph_rag/fuse.py +371 -0
  74. brain/graph_rag/global_.py +412 -0
  75. brain/graph_rag/grouping.py +372 -0
  76. brain/graph_rag/person_resolver.py +167 -0
  77. brain/graph_rag/reconcile.py +792 -0
  78. brain/graph_rag/relational.py +353 -0
  79. brain/graph_rag/retrieve.py +526 -0
  80. brain/graph_rag/router.py +288 -0
  81. brain/graph_rag/schema.py +320 -0
  82. brain/graph_rag/sync.py +237 -0
  83. brain/graph_rag/tenancy.py +43 -0
  84. brain/graph_rag/themes.py +501 -0
  85. brain/graph_rag/weighting.py +202 -0
  86. brain/ingest/__init__.py +1926 -0
  87. brain/ingest/chunker.py +249 -0
  88. brain/ingest/docx.py +40 -0
  89. brain/ingest/gmail.py +621 -0
  90. brain/ingest/markdown.py +37 -0
  91. brain/ingest/pdf.py +61 -0
  92. brain/ingest/stdin.py +22 -0
  93. brain/ingest/sub_tokens.py +91 -0
  94. brain/ingest/text.py +16 -0
  95. brain/interactions.py +205 -0
  96. brain/maintenance.py +355 -0
  97. brain/mcp_server.py +3405 -0
  98. brain/migrations/001_init.sql +43 -0
  99. brain/migrations/002_qwen3_embedding.sql +17 -0
  100. brain/migrations/003_vault_model.sql +41 -0
  101. brain/migrations/004_relax_content_hash_uniqueness.sql +18 -0
  102. brain/migrations/005_derived_links.sql +67 -0
  103. brain/migrations/006_dedup_file_by_source_path.sql +25 -0
  104. brain/migrations/007_email_thread_and_draft.sql +15 -0
  105. brain/migrations/008_gmail_thread_unique.sql +11 -0
  106. brain/migrations/009_chunks_weighted_tsv.sql +28 -0
  107. brain/migrations/010_interactions.sql +30 -0
  108. brain/migrations/011_documents_summary.sql +23 -0
  109. brain/migrations/012_graphrag.sql +171 -0
  110. brain/migrations/013_graphrag_communities.sql +125 -0
  111. brain/migrations/014_graphrag_community_summary_hash.sql +33 -0
  112. brain/migrations/015_interactions_graph_targets.sql +89 -0
  113. brain/migrations/016_index_hygiene.sql +61 -0
  114. brain/migrations/017_elicit.sql +30 -0
  115. brain/migrations/018_review_gap_signal_kinds.sql +40 -0
  116. brain/migrations/019_search_queries.sql +35 -0
  117. brain/migrations/020_link_suggestions.sql +40 -0
  118. brain/migrations/021_timeline_doc_date.sql +34 -0
  119. brain/migrations/022_link_suggestions_undirected.sql +84 -0
  120. brain/migrations/023_search_queries_fts_count.sql +28 -0
  121. brain/quartz_overrides/__init__.py +8 -0
  122. brain/quartz_overrides/quartz/bootstrap-cli.mjs +65 -0
  123. brain/quartz_overrides/quartz/build.ts +568 -0
  124. brain/quartz_overrides/quartz/cli/args.js +152 -0
  125. brain/quartz_overrides/quartz/cli/build_partial_handler.js +544 -0
  126. brain/quartz_overrides/quartz/cli/handlers.js +636 -0
  127. brain/quartz_overrides/quartz/components/CommandPalette.tsx +172 -0
  128. brain/quartz_overrides/quartz/components/Explorer.tsx +198 -0
  129. brain/quartz_overrides/quartz/components/Footer.tsx +27 -0
  130. brain/quartz_overrides/quartz/components/Graph.tsx +468 -0
  131. brain/quartz_overrides/quartz/components/PageTitle.tsx +72 -0
  132. brain/quartz_overrides/quartz/components/RelatedDocs.tsx +38 -0
  133. brain/quartz_overrides/quartz/components/Search.tsx +161 -0
  134. brain/quartz_overrides/quartz/components/SummaryLede.tsx +72 -0
  135. brain/quartz_overrides/quartz/components/index.ts +92 -0
  136. brain/quartz_overrides/quartz/components/pages/TagContent.tsx +272 -0
  137. brain/quartz_overrides/quartz/components/scripts/commandPalette.inline.ts +665 -0
  138. brain/quartz_overrides/quartz/components/scripts/explorer.inline.ts +768 -0
  139. brain/quartz_overrides/quartz/components/scripts/graph.inline.ts +2302 -0
  140. brain/quartz_overrides/quartz/components/scripts/relatedDocs.inline.ts +163 -0
  141. brain/quartz_overrides/quartz/components/scripts/search.inline.ts +1011 -0
  142. brain/quartz_overrides/quartz/plugins/emitters/contentIndex.ts +546 -0
  143. brain/quartz_overrides/quartz/plugins/transformers/codeCopy.ts +94 -0
  144. brain/quartz_overrides/quartz/plugins/transformers/derivedFenceMark.ts +302 -0
  145. brain/quartz_overrides/quartz/plugins/transformers/emailThread.ts +148 -0
  146. brain/quartz_overrides/quartz/plugins/transformers/emptyDoorFilter.ts +213 -0
  147. brain/quartz_overrides/quartz/plugins/transformers/index.ts +114 -0
  148. brain/quartz_overrides/quartz/plugins/transformers/linkKindMark.ts +205 -0
  149. brain/quartz_overrides/quartz/plugins/transformers/linkSourceTag.ts +104 -0
  150. brain/quartz_overrides/quartz/plugins/transformers/relativeDate.ts +100 -0
  151. brain/quartz_overrides/quartz/plugins/transformers/reloadSignal.ts +131 -0
  152. brain/quartz_overrides/quartz/processors/parse.ts +371 -0
  153. brain/quartz_overrides/quartz/processors/parser_cache.ts +78 -0
  154. brain/quartz_overrides/quartz/static/brain-logo-dark.png +0 -0
  155. brain/quartz_overrides/quartz/static/brain-logo-light.png +0 -0
  156. brain/quartz_overrides/quartz/static/codeCopy.js +196 -0
  157. brain/quartz_overrides/quartz/static/emailThread.js +334 -0
  158. brain/quartz_overrides/quartz/static/favicon.ico +0 -0
  159. brain/quartz_overrides/quartz/static/icon.png +0 -0
  160. brain/quartz_overrides/quartz/static/linkSourceTag.js +104 -0
  161. brain/quartz_overrides/quartz/static/relativeDate.js +142 -0
  162. brain/quartz_overrides/quartz/static/reload.js +168 -0
  163. brain/quartz_overrides/quartz/styles/brain/_article.scss +252 -0
  164. brain/quartz_overrides/quartz/styles/brain/_atmosphere.scss +113 -0
  165. brain/quartz_overrides/quartz/styles/brain/_callouts.scss +180 -0
  166. brain/quartz_overrides/quartz/styles/brain/_cmdk.scss +7 -0
  167. brain/quartz_overrides/quartz/styles/brain/_code.scss +208 -0
  168. brain/quartz_overrides/quartz/styles/brain/_command_palette.scss +369 -0
  169. brain/quartz_overrides/quartz/styles/brain/_email_thread.scss +228 -0
  170. brain/quartz_overrides/quartz/styles/brain/_explorer.scss +142 -0
  171. brain/quartz_overrides/quartz/styles/brain/_home.scss +182 -0
  172. brain/quartz_overrides/quartz/styles/brain/_links.scss +322 -0
  173. brain/quartz_overrides/quartz/styles/brain/_marginalia.scss +117 -0
  174. brain/quartz_overrides/quartz/styles/brain/_motion.scss +175 -0
  175. brain/quartz_overrides/quartz/styles/brain/_people_hub.scss +100 -0
  176. brain/quartz_overrides/quartz/styles/brain/_related_docs.scss +137 -0
  177. brain/quartz_overrides/quartz/styles/brain/_search.scss +252 -0
  178. brain/quartz_overrides/quartz/styles/brain/_sidebar.scss +468 -0
  179. brain/quartz_overrides/quartz/styles/brain/_summary_lede.scss +56 -0
  180. brain/quartz_overrides/quartz/styles/brain/_surface.scss +43 -0
  181. brain/quartz_overrides/quartz/styles/brain/_tag_content.scss +118 -0
  182. brain/quartz_overrides/quartz/styles/brain/_tokens.scss +197 -0
  183. brain/quartz_overrides/quartz/styles/brain/_typography.scss +92 -0
  184. brain/quartz_overrides/quartz/styles/custom.scss +89 -0
  185. brain/quartz_overrides/quartz/styles/graph.scss +505 -0
  186. brain/quartz_overrides/quartz/util/ctx.ts +92 -0
  187. brain/quartz_overrides/quartz/util/fastpath_manifest.ts +608 -0
  188. brain/quartz_overrides/quartz/util/path.ts +358 -0
  189. brain/quartz_overrides/quartz/util/sourceIcons.ts +55 -0
  190. brain/quartz_overrides/quartz.config.ts +270 -0
  191. brain/quartz_overrides/quartz.layout.ts +314 -0
  192. brain/queries.py +1188 -0
  193. brain/rank_fusion.py +8 -0
  194. brain/resurface.py +210 -0
  195. brain/review/__init__.py +26 -0
  196. brain/review/emit.py +27 -0
  197. brain/review/queries.py +436 -0
  198. brain/review/render.py +196 -0
  199. brain/review/scans.py +355 -0
  200. brain/review/weekly.py +413 -0
  201. brain/search.py +704 -0
  202. brain/set_similarity.py +15 -0
  203. brain/setup.py +1205 -0
  204. brain/tags.py +56 -0
  205. brain/templates/Caddyfile.j2 +9 -0
  206. brain/templates/__init__.py +1 -0
  207. brain/templates/bin/__init__.py +1 -0
  208. brain/templates/bin/_brain-brief-fg.sh +25 -0
  209. brain/templates/bin/_brain-build-fg.sh +53 -0
  210. brain/templates/bin/_brain-watcher-fg.sh +65 -0
  211. brain/templates/bin/brain-down.sh +89 -0
  212. brain/templates/bin/brain-status.sh +83 -0
  213. brain/templates/bin/brain-up.sh +221 -0
  214. brain/templates/docker/age/Dockerfile +79 -0
  215. brain/templates/docker-compose.stock.yml.j2 +26 -0
  216. brain/templates/docker-compose.yml.j2 +34 -0
  217. brain/templates/env.example +190 -0
  218. brain/templates/launchd/__init__.py +1 -0
  219. brain/templates/launchd/com.brain.brief.plist.j2 +45 -0
  220. brain/templates/launchd/com.brain.build.plist.j2 +46 -0
  221. brain/templates/launchd/com.brain.watcher.plist.j2 +46 -0
  222. brain/templates/skill/SKILL.md +63 -0
  223. brain/templates/skill/__init__.py +1 -0
  224. brain/timeline.py +834 -0
  225. brain/todo.py +124 -0
  226. brain/uninstall.py +185 -0
  227. brain/vault/__init__.py +115 -0
  228. brain/vault/_atomic.py +25 -0
  229. brain/vault/daily_index.py +228 -0
  230. brain/vault/derived_links/__init__.py +50 -0
  231. brain/vault/derived_links/directory.py +683 -0
  232. brain/vault/derived_links/fence.py +408 -0
  233. brain/vault/derived_links/gws.py +64 -0
  234. brain/vault/derived_links/participants.py +143 -0
  235. brain/vault/derived_links/pass_runner.py +362 -0
  236. brain/vault/derived_links/rules.py +137 -0
  237. brain/vault/export.py +683 -0
  238. brain/vault/frontmatter.py +165 -0
  239. brain/vault/graph.py +620 -0
  240. brain/vault/graph_format.py +388 -0
  241. brain/vault/link_rewrite.py +235 -0
  242. brain/vault/links.py +260 -0
  243. brain/vault/note_builder.py +211 -0
  244. brain/vault/paths.py +55 -0
  245. brain/vault/quartz_overlay.py +236 -0
  246. brain/vault/rename.py +591 -0
  247. brain/vault/resolver.py +304 -0
  248. brain/vault/slug.py +127 -0
  249. brain/vault/sync.py +1513 -0
  250. brain/vault/sync_summaries.py +264 -0
  251. brain/vault/templates.py +145 -0
  252. brain/vault/watch.py +1052 -0
  253. brain/wiki/__init__.py +6 -0
  254. brain/wiki/_github_slugger.py +76 -0
  255. brain/wiki/_person_name.py +314 -0
  256. brain/wiki/build_homepage.py +541 -0
  257. brain/wiki/build_partial.py +273 -0
  258. brain/wiki/build_people.py +934 -0
  259. brain/wiki/build_related.py +758 -0
  260. brain/wiki/build_swap.py +585 -0
  261. brain/wiki/build_watcher.py +975 -0
  262. brain/wiki/edit_classifier.py +215 -0
  263. brain/wiki/errors.py +10 -0
  264. brain/wiki/fastpath_manifest.py +475 -0
  265. brain/wiki/fastpath_state.py +174 -0
  266. brain/wiki/install.py +296 -0
  267. brain/wiki/slug.py +111 -0
  268. secondbrain_py-0.2.1.dist-info/METADATA +195 -0
  269. secondbrain_py-0.2.1.dist-info/RECORD +273 -0
  270. secondbrain_py-0.2.1.dist-info/WHEEL +5 -0
  271. secondbrain_py-0.2.1.dist-info/entry_points.txt +11 -0
  272. secondbrain_py-0.2.1.dist-info/licenses/LICENSE +21 -0
  273. secondbrain_py-0.2.1.dist-info/top_level.txt +1 -0
brain/vault/export.py ADDED
@@ -0,0 +1,683 @@
1
+ """One-shot DB → vault export.
2
+
3
+ Walks ``documents`` in batches and materializes each row as a Markdown file
4
+ with YAML frontmatter under the vault root. Idempotent: a re-run skips files
5
+ whose existing content_hash on disk matches the document's content_hash in
6
+ the DB.
7
+
8
+ Older manual ingests have no ``sources`` row (no upstream identifier). For
9
+ those we synthesize a ``local-<short-uuid>`` placeholder in the filename so
10
+ two such docs with the same date + title can still be disambiguated.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import hashlib
15
+ from collections.abc import Iterator
16
+ from dataclasses import dataclass, field
17
+ from datetime import date, datetime
18
+ from pathlib import Path
19
+ from typing import Any
20
+
21
+ import psycopg
22
+ import yaml
23
+
24
+ from . import init_vault
25
+ from ._atomic import atomic_write_text
26
+ from .frontmatter import dump_frontmatter, parse_frontmatter
27
+ from .slug import gmail_slug, slugify
28
+
29
+ _BATCH_SIZE = 100
30
+ _MAX_SHORT_ID = 8 # first N chars of an external/document UUID for filename use
31
+ _EXPORT_OWNED_FRONTMATTER_KEYS = frozenset(
32
+ {
33
+ "id",
34
+ "title",
35
+ "tags",
36
+ "aliases",
37
+ "kind",
38
+ "content_type",
39
+ "source",
40
+ "external_id",
41
+ "created",
42
+ "updated",
43
+ "vault_path",
44
+ # NOTE: ``summary`` is intentionally NOT in this strip set.
45
+ # Wave Q2-SUMMARY-WIKI's first cut stripped it always (the
46
+ # documented theory was that vault-tier hand-authored
47
+ # ``summary:`` lines round-trip through ``documents.summary``).
48
+ # That theory was wrong — sync's ``_build_metadata`` puts every
49
+ # non-reserved frontmatter key into ``documents.metadata`` for
50
+ # vault-tier rows, so a user's ``summary:`` lands in
51
+ # ``metadata["summary"]``, NOT in ``documents.summary``. Stripping
52
+ # it here deleted the user's content on the next export.
53
+ # Codex finding 2 (2026-05-11) caught the regression. The fix:
54
+ # leave ``summary`` in freeform passthrough for vault-tier; let
55
+ # :func:`_build_frontmatter` resolve precedence (Q1-D
56
+ # ``documents.summary`` wins; vault-tier metadata.summary serves
57
+ # as the fallback when ``documents.summary`` is NULL).
58
+ }
59
+ )
60
+
61
+
62
+ @dataclass
63
+ class ExportSummary:
64
+ """Outcome counts for ``brain vault export``."""
65
+
66
+ written: int = 0
67
+ skipped: int = 0
68
+ errors: list[str] = field(default_factory=list)
69
+
70
+
71
+ @dataclass
72
+ class _DocumentForExport:
73
+ """Internal projection of a documents row plus its source kind/external id."""
74
+
75
+ id: str
76
+ title: str
77
+ content: str
78
+ content_hash: str
79
+ content_type: str
80
+ tags: list[str]
81
+ metadata: dict[str, Any]
82
+ ingested_at: datetime | None
83
+ kind: str
84
+ vault_path: str | None
85
+ source_kind: str | None
86
+ source_external_id: str | None
87
+ draft: bool
88
+ # Wave Q2-SUMMARY-WIKI: ``documents.summary`` written by the Q1-D
89
+ # ``OllamaEnricher`` post-ingest hook. Plumbed onto the export
90
+ # projection so :func:`_build_frontmatter` can emit it as a wiki-
91
+ # readable field. ``None`` for any doc the enricher hasn't touched
92
+ # yet (short docs, Ollama unavailable, ``enrich=False``).
93
+ summary: str | None
94
+
95
+
96
+ def _is_directory_unmanaged(target: Path) -> bool:
97
+ """True iff ``target`` is non-empty AND not a previously-initialized vault.
98
+
99
+ A "managed" vault is one that already contains the ``README.md`` written
100
+ by :func:`init_vault`. Anything else with files in it is unmanaged — the
101
+ user has to opt in with ``--force`` so we never silently scribble into
102
+ an unrelated folder.
103
+
104
+ Dotfiles and hidden directories (``.git/``, ``.DS_Store``, etc.) are
105
+ intentionally excluded from the emptiness check. A folder that contains
106
+ only a ``.git`` directory still counts as empty for our purposes — the
107
+ user almost certainly wants to use a git-tracked vault, and forcing
108
+ them through ``--force`` for an empty initial repo would be obnoxious.
109
+ """
110
+ if not target.exists():
111
+ return False
112
+ if not target.is_dir():
113
+ return True
114
+ contents = [p for p in target.iterdir() if not p.name.startswith(".")]
115
+ if not contents:
116
+ return False
117
+ # Treat the presence of vault README.md (signature of init_vault) as the
118
+ # "yes, this is our folder" marker.
119
+ return not (target / "README.md").is_file()
120
+
121
+
122
+ _DOCUMENT_FOR_EXPORT_COLUMNS = (
123
+ "d.id::text, d.title, d.content, d.content_hash, "
124
+ "d.content_type, d.tags, d.metadata, d.ingested_at, "
125
+ "d.kind, d.vault_path, s.kind, s.external_id, d.draft, "
126
+ # Wave Q2-SUMMARY-WIKI: pull the Q1-D auto-summary onto the export
127
+ # projection so :func:`_build_frontmatter` can emit it as a
128
+ # ``summary:`` field in the per-doc mirror.
129
+ "d.summary"
130
+ )
131
+
132
+
133
+ def _row_to_document_for_export(row: tuple[Any, ...]) -> _DocumentForExport:
134
+ """Project a SELECT row into the dataclass.
135
+
136
+ Centralizes the column-index mapping so :func:`_iter_documents` and
137
+ :func:`_fetch_document_for_export` share one source of truth and a
138
+ schema change touches a single helper.
139
+ """
140
+ return _DocumentForExport(
141
+ id=str(row[0]),
142
+ title=str(row[1]),
143
+ content=str(row[2]),
144
+ content_hash=str(row[3]),
145
+ content_type=str(row[4]),
146
+ tags=list(row[5] or []),
147
+ metadata=dict(row[6] or {}),
148
+ ingested_at=row[7],
149
+ kind=str(row[8]),
150
+ vault_path=row[9],
151
+ source_kind=row[10],
152
+ source_external_id=row[11],
153
+ draft=bool(row[12]),
154
+ # ``documents.summary`` is NULL until the Q1-D enrich hook
155
+ # populates it (or a future ``brain enrich --backfill`` run).
156
+ # Coerce to ``str`` only when set so ``_build_frontmatter`` can
157
+ # gate the emit on a truthy check.
158
+ summary=str(row[13]) if row[13] is not None else None,
159
+ )
160
+
161
+
162
+ def _iter_documents(conn: psycopg.Connection[Any]) -> Iterator[_DocumentForExport]:
163
+ """Yield every document, joined with its source row, in batches.
164
+
165
+ Keyset pagination on ``documents.id`` keeps memory bounded even for very
166
+ large corpora; the batch is materialized in Python (the connection is
167
+ free for follow-up writes / lookups while the iterator is in flight).
168
+ """
169
+ last_id: str | None = None
170
+ while True:
171
+ if last_id is None:
172
+ rows = conn.execute(
173
+ f"""
174
+ SELECT {_DOCUMENT_FOR_EXPORT_COLUMNS}
175
+ FROM documents d
176
+ LEFT JOIN sources s ON s.id = d.source_id
177
+ ORDER BY d.id
178
+ LIMIT %s
179
+ """,
180
+ (_BATCH_SIZE,),
181
+ ).fetchall()
182
+ else:
183
+ rows = conn.execute(
184
+ f"""
185
+ SELECT {_DOCUMENT_FOR_EXPORT_COLUMNS}
186
+ FROM documents d
187
+ LEFT JOIN sources s ON s.id = d.source_id
188
+ WHERE d.id > %s::uuid
189
+ ORDER BY d.id
190
+ LIMIT %s
191
+ """,
192
+ (last_id, _BATCH_SIZE),
193
+ ).fetchall()
194
+ if not rows:
195
+ return
196
+ last_id = str(rows[-1][0])
197
+ for r in rows:
198
+ yield _row_to_document_for_export(r)
199
+
200
+
201
+ def _fetch_document_for_export(
202
+ conn: psycopg.Connection[Any], document_id: str
203
+ ) -> _DocumentForExport | None:
204
+ """Fetch a single ``documents`` row in the export projection.
205
+
206
+ Returns ``None`` when no row matches ``document_id``. Mirrors the column
207
+ list used by :func:`_iter_documents` so the resulting dataclass is
208
+ populated identically for both code paths.
209
+ """
210
+ row = conn.execute(
211
+ f"""
212
+ SELECT {_DOCUMENT_FOR_EXPORT_COLUMNS}
213
+ FROM documents d
214
+ LEFT JOIN sources s ON s.id = d.source_id
215
+ WHERE d.id = %s::uuid
216
+ """,
217
+ (document_id,),
218
+ ).fetchone()
219
+ if row is None:
220
+ return None
221
+ return _row_to_document_for_export(row)
222
+
223
+
224
+ def _short_id(value: str) -> str:
225
+ """Return the first 8 hex chars of a UUID-like value (no surrounding dashes).
226
+
227
+ Used both for the external-id slot in filenames and for the collision
228
+ suffix. Stripping dashes first makes ``-aabbccdd`` collisions deterministic
229
+ regardless of the full UUID's hyphenation style. For non-UUID
230
+ ``external_id`` values (e.g. Slack's ``1234567.890`` timestamps) the
231
+ strip-then-truncate behavior preserves the leading bytes verbatim once
232
+ any non-hex separator characters have been dropped.
233
+ """
234
+ return value.replace("-", "")[:_MAX_SHORT_ID]
235
+
236
+
237
+ def _date_prefix(doc: _DocumentForExport) -> str:
238
+ """Pick the YYYY-MM-DD prefix for an ingested file's filename.
239
+
240
+ ``metadata.date`` wins when present (a string ISO date or datetime, or a
241
+ date/datetime object). Otherwise we fall back to ``ingested_at::date``,
242
+ which is non-NULL by schema (``documents.ingested_at NOT NULL DEFAULT
243
+ NOW()``) — the assert is defensive so a future schema regression
244
+ surfaces here rather than producing a malformed filename.
245
+ """
246
+ raw = doc.metadata.get("date")
247
+ if isinstance(raw, str) and raw:
248
+ # Accept "2026-04-15" or "2026-04-15T..." — the date is the first 10 chars.
249
+ return raw[:10]
250
+ if isinstance(raw, (datetime, date)):
251
+ return raw.strftime("%Y-%m-%d") if isinstance(raw, date) else raw.date().isoformat()
252
+ assert doc.ingested_at is not None, "documents.ingested_at is NOT NULL"
253
+ return doc.ingested_at.date().isoformat()
254
+
255
+
256
+ def _parse_iso_datetime(raw: Any) -> datetime | None:
257
+ """Parse an ISO-8601 string (with optional trailing ``Z``) into a datetime.
258
+
259
+ Returns ``None`` for any non-string / unparseable input. Used by the
260
+ Gmail-slug branch to read ``metadata.sent_at`` (set by the gmail
261
+ extractor). Naive datetimes survive as-is — :func:`gmail_slug` treats
262
+ them as UTC.
263
+ """
264
+ if not isinstance(raw, str) or not raw.strip():
265
+ return None
266
+ try:
267
+ return datetime.fromisoformat(raw.replace("Z", "+00:00"))
268
+ except ValueError:
269
+ return None
270
+
271
+
272
+ def _gmail_relative_path(doc: _DocumentForExport) -> str | None:
273
+ """Compute the ``_ingested/gmail/<gmail-slug>.md`` path for a Gmail doc.
274
+
275
+ Returns ``None`` when the metadata is too sparse to build a stable slug
276
+ (no ``thread_id`` AND no usable date) — the caller falls back to the
277
+ generic ingested-path rules so legacy rows still export. Idempotent: a
278
+ given ``(thread_id, sent_at, subject)`` triple always yields the same
279
+ path.
280
+ """
281
+ thread_id = doc.metadata.get("thread_id")
282
+ if not isinstance(thread_id, str) or not thread_id:
283
+ return None
284
+ sent_at = _parse_iso_datetime(doc.metadata.get("sent_at"))
285
+ if sent_at is None and doc.ingested_at is None:
286
+ return None
287
+ slug = gmail_slug(
288
+ thread_id,
289
+ sent_at,
290
+ doc.title,
291
+ fallback_date=doc.ingested_at,
292
+ )
293
+ return f"_ingested/gmail/{slug}.md"
294
+
295
+
296
+ def _ingested_relative_path(
297
+ doc: _DocumentForExport, used_paths: set[str]
298
+ ) -> str:
299
+ """Compute the ``_ingested/<source>/...`` path for an ingested-tier doc.
300
+
301
+ The shape depends on the source:
302
+
303
+ - ``manual``: ``_ingested/manual/<slug>.md`` (no date prefix; older
304
+ manual ingests had no upstream date at all).
305
+ - ``gmail``: ``_ingested/gmail/<gmail-slug>.md`` where ``<gmail-slug>``
306
+ is :func:`brain.vault.slug.gmail_slug` over
307
+ ``(thread_id, sent_at, subject)``. Stable across re-ingests of the
308
+ same thread; falls back to the generic shape below for legacy rows
309
+ missing ``thread_id``.
310
+ - everything else: ``_ingested/<source>/<YYYY-MM-DD>-<external-id>-<slug>.md``
311
+ using the first 8 chars of ``sources.external_id`` (or
312
+ ``local-<short-doc-id>`` when no source row exists, e.g. legacy
313
+ manual ingests under any kind).
314
+
315
+ On collision (two docs hashing to the same path) we append the short
316
+ document UUID — stable across re-runs because it's keyed off the immutable
317
+ ``documents.id``.
318
+ """
319
+ source = doc.source_kind or "manual"
320
+ slug = slugify(doc.title)
321
+
322
+ if source == "manual":
323
+ candidate = f"_ingested/manual/{slug}.md"
324
+ elif source == "gmail":
325
+ gmail_path = _gmail_relative_path(doc)
326
+ if gmail_path is not None:
327
+ candidate = gmail_path
328
+ else:
329
+ date_prefix = _date_prefix(doc)
330
+ external = (
331
+ _short_id(doc.source_external_id)
332
+ if doc.source_external_id
333
+ else f"local-{_short_id(doc.id)}"
334
+ )
335
+ candidate = f"_ingested/gmail/{date_prefix}-{external}-{slug}.md"
336
+ else:
337
+ date_prefix = _date_prefix(doc)
338
+ external = (
339
+ _short_id(doc.source_external_id)
340
+ if doc.source_external_id
341
+ else f"local-{_short_id(doc.id)}"
342
+ )
343
+ candidate = f"_ingested/{source}/{date_prefix}-{external}-{slug}.md"
344
+
345
+ if candidate in used_paths:
346
+ # Deterministic collision suffix: short hash of the doc id, never a
347
+ # counter (counters would shuffle every re-run).
348
+ suffix = _short_id(doc.id)
349
+ stem, ext = candidate.rsplit(".", 1)
350
+ candidate = f"{stem}-{suffix}.{ext}"
351
+ return candidate
352
+
353
+
354
+ def _resolve_relative_path(
355
+ doc: _DocumentForExport, used_paths: set[str]
356
+ ) -> str:
357
+ """Pick the on-disk path for ``doc``.
358
+
359
+ Rows with an explicit ``vault_path`` round-trip identity (vault-tier files
360
+ stay where the user authored them; ingested mirrors stay at their recorded
361
+ canonical path). Rows without one land in ``_ingested/`` per
362
+ :func:`_ingested_relative_path`.
363
+ """
364
+ if doc.vault_path:
365
+ return doc.vault_path
366
+ return _ingested_relative_path(doc, used_paths)
367
+
368
+
369
+ def _build_frontmatter(doc: _DocumentForExport) -> dict[str, Any]:
370
+ """Return the ordered frontmatter mapping for ``doc``.
371
+
372
+ Field order is intentional and stable: id, title, created, updated, tags,
373
+ aliases (when non-empty), kind, content_type, ``summary`` (when the Q1-D
374
+ enricher has populated ``documents.summary``), then ingested-tier extras
375
+ (source / external_id) when present, and finally ``draft: true`` when the
376
+ document is quarantined. Anything not applicable (e.g. ingested extras
377
+ for a vault-tier doc, an empty alias list, or a NULL summary) is omitted
378
+ entirely rather than written as ``null`` / ``[]`` — keeps the vault file
379
+ readable and round-trips cleanly through sync.
380
+
381
+ The ``draft`` line is emitted ONLY when ``documents.draft`` is true. The
382
+ default-false case is the vast majority of files; writing ``draft: false``
383
+ on every export would be visual noise. The Quartz contentIndex emitter
384
+ reads this key (or the absence of it) to filter quarantined docs out of
385
+ the wiki's explorer / graph / search index without touching the DB row.
386
+
387
+ The ``summary`` line (wave Q2-SUMMARY-WIKI) is emitted whenever
388
+ ``documents.summary`` is non-NULL — that is the same surface
389
+ ``brain show`` / MCP ``brain_show`` expose. The Quartz
390
+ ``SummaryLede`` component renders it as an inline TL;DR above the
391
+ article body. Summaries are NOT indexed (the Q1-D spec keeps them
392
+ out of tsv / embeddings / SearchExplanation); this writer is the
393
+ read-surface pipeline only.
394
+
395
+ Vault-tier rows also preserve freeform metadata keys because sync stores
396
+ user-authored frontmatter there. Export-owned keys stay canonical here,
397
+ and ingested-tier metadata remains DB-only so source metadata does not
398
+ leak into mirror frontmatter.
399
+
400
+ Aliases come from ``documents.metadata['aliases']`` (the canonical
401
+ storage location per the spec) — only string elements are emitted; any
402
+ non-string value in the array is dropped silently to keep the
403
+ frontmatter's ``aliases:`` always be a flat list of strings.
404
+ """
405
+ fields: dict[str, Any] = {
406
+ "id": doc.id,
407
+ "title": doc.title,
408
+ }
409
+ if doc.ingested_at is not None:
410
+ iso = doc.ingested_at.isoformat()
411
+ fields["created"] = iso
412
+ fields["updated"] = iso
413
+ fields["tags"] = list(doc.tags)
414
+ aliases = _aliases_from_metadata(doc.metadata)
415
+ if aliases:
416
+ fields["aliases"] = aliases
417
+ fields["kind"] = doc.kind
418
+ fields["content_type"] = doc.content_type
419
+ # Wave Q2-SUMMARY-WIKI: emit ``summary`` with tier-aware precedence.
420
+ # 1. ``documents.summary`` (Q1-D ``OllamaEnricher``) — authoritative
421
+ # whenever non-NULL. The wiki's ``SummaryLede`` reads this key.
422
+ # 2. For vault-tier docs only, fall back to a hand-authored
423
+ # ``summary:`` line that sync stored in ``documents.metadata``.
424
+ # Ingested-tier docs never fall back — their metadata is
425
+ # DB-authoritative, not user-authored.
426
+ # The early write here pins the field's position right after
427
+ # ``content_type`` so YAML field order stays stable regardless of
428
+ # which source the value came from. The later vault-tier freeform
429
+ # merge may overwrite this with the metadata value; we re-assert
430
+ # ``documents.summary`` after the merge to keep it authoritative.
431
+ effective_summary: str | None = doc.summary
432
+ if effective_summary is None and doc.kind == "vault":
433
+ metadata_summary = doc.metadata.get("summary")
434
+ if isinstance(metadata_summary, str) and metadata_summary:
435
+ effective_summary = metadata_summary
436
+ if effective_summary:
437
+ fields["summary"] = effective_summary
438
+ if doc.kind == "ingested":
439
+ if doc.source_kind:
440
+ fields["source"] = doc.source_kind
441
+ if doc.source_external_id:
442
+ fields["external_id"] = doc.source_external_id
443
+ if doc.kind == "vault":
444
+ fields.update(_freeform_vault_metadata(doc.metadata))
445
+ # Re-assert Q1-D precedence: the freeform merge above may have
446
+ # overwritten ``fields["summary"]`` with ``metadata["summary"]``
447
+ # (Codex finding 2 fix removed ``summary`` from the strip set so
448
+ # vault-tier hand-authored values survive). When
449
+ # ``documents.summary`` is also set, it wins — re-pin it at the
450
+ # field's existing dict position so YAML order is unchanged.
451
+ if doc.summary:
452
+ fields["summary"] = doc.summary
453
+ if doc.draft:
454
+ fields["draft"] = True
455
+ return fields
456
+
457
+
458
+ def _freeform_vault_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
459
+ """Return vault-tier metadata keys that should round-trip to YAML."""
460
+ out: dict[str, Any] = {}
461
+ for key, value in metadata.items():
462
+ if key in _EXPORT_OWNED_FRONTMATTER_KEYS:
463
+ continue
464
+ out[key] = value
465
+ return out
466
+
467
+
468
+ def _aliases_from_metadata(metadata: dict[str, Any]) -> list[str]:
469
+ """Extract a clean list of alias strings from ``documents.metadata``.
470
+
471
+ Only emits the field when the metadata payload actually carries one or
472
+ more string aliases. Anything else (missing key, non-list, list of
473
+ non-strings) returns an empty list — the caller then omits the
474
+ ``aliases:`` line altogether. Defensive coercion here keeps a corrupt
475
+ metadata blob from poisoning the entire export pass.
476
+ """
477
+ raw = metadata.get("aliases")
478
+ if not isinstance(raw, list):
479
+ return []
480
+ return [a for a in raw if isinstance(a, str) and a]
481
+
482
+
483
+ def _content_hash(text: str) -> str:
484
+ """SHA-256 of ``text`` — matches the ingest pipeline's hashing scheme.
485
+
486
+ Idempotent re-export compares this against ``documents.content_hash`` on a
487
+ body re-extracted from the existing vault file (frontmatter stripped) so
488
+ the user can edit frontmatter (or run a re-export with a new field
489
+ ordering) without forcing a re-write.
490
+ """
491
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
492
+
493
+
494
+ def _existing_body_hash(target: Path) -> str | None:
495
+ """Read ``target`` as a vault file and return SHA-256 of its body.
496
+
497
+ Returns ``None`` if the file doesn't exist or can't be parsed (we'll
498
+ treat unparseable files as "needs rewrite" rather than skipping them
499
+ silently). Frontmatter is stripped before hashing so a frontmatter-only
500
+ edit doesn't trigger a needless body rewrite.
501
+ """
502
+ if not target.is_file():
503
+ return None
504
+ try:
505
+ text = target.read_text(encoding="utf-8")
506
+ except OSError:
507
+ return None
508
+ try:
509
+ _, body = parse_frontmatter(text)
510
+ except (ValueError, yaml.YAMLError):
511
+ # ValueError covers our own "frontmatter must be a mapping" guard;
512
+ # yaml.YAMLError covers malformed YAML inside the fences. Anything
513
+ # broader would mask real bugs (per CLAUDE.md: always catch specific
514
+ # exceptions).
515
+ return None
516
+ return _content_hash(body)
517
+
518
+
519
+ def _write_doc_file(
520
+ doc: _DocumentForExport,
521
+ *,
522
+ vault_path: Path,
523
+ relative: str,
524
+ force: bool = False,
525
+ ) -> tuple[Path, bool]:
526
+ """Materialize ``doc`` at ``vault_path / relative`` and return the result.
527
+
528
+ The returned tuple is ``(target_path, written)`` — ``written`` is False
529
+ when the existing file's body already matches ``doc.content_hash``
530
+ (idempotent skip), True when the file was rewritten or freshly created.
531
+
532
+ ``force=True`` bypasses the body-hash skip and rewrites the file even
533
+ when the body is unchanged. Callers must use ``force=True`` when they've
534
+ already determined a frontmatter-only change happened — otherwise the
535
+ body-hash check (which only fingerprints the body) silently drops the
536
+ rewrite and leaves stale frontmatter on disk. The default
537
+ ``force=False`` keeps the corpus-dump path (:func:`export_vault`) cheap
538
+ on re-runs.
539
+
540
+ Raises :class:`OSError` if the write fails. Callers decide whether to
541
+ aggregate the failure into a summary or surface it directly.
542
+ """
543
+ target = vault_path / relative
544
+ if not force:
545
+ existing_hash = _existing_body_hash(target)
546
+ if existing_hash == doc.content_hash:
547
+ return target, False
548
+
549
+ target.parent.mkdir(parents=True, exist_ok=True)
550
+ fields = _build_frontmatter(doc)
551
+ text = dump_frontmatter(fields, doc.content)
552
+ # Atomic rewrite — sibling tempfile + ``os.replace`` so a crash between
553
+ # the write and the rename never leaves a truncated mirror file on disk.
554
+ # All writeback callers (``brain edit`` / ``brain tag`` / ``brain mark-draft`` /
555
+ # ``brain vault sync-summaries`` / full ``brain vault export``) funnel through
556
+ # here, so this single line gives the whole mirror-write surface crash
557
+ # safety. Matches the convention used by every other body-rewriter under
558
+ # ``vault/`` (see :mod:`brain.vault._atomic`).
559
+ atomic_write_text(target, text)
560
+ return target, True
561
+
562
+
563
+ def export_vault(
564
+ conn: psycopg.Connection[Any],
565
+ *,
566
+ vault_path: Path,
567
+ force: bool = False,
568
+ ) -> ExportSummary:
569
+ """Dump every ``documents`` row to ``vault_path`` as a Markdown file.
570
+
571
+ Steps:
572
+
573
+ 1. Validate the target — refuse to write into a non-empty unmanaged
574
+ directory unless ``force`` is true (keeps us from scribbling into an
575
+ arbitrary folder the user pointed us at by mistake).
576
+ 2. Run :func:`init_vault` semantics so ``_templates/``, ``_ingested/``,
577
+ and the README scaffold are guaranteed to exist.
578
+ 3. Iterate documents in keyset-paginated batches; for each one, compute
579
+ the destination path, build the frontmatter, and skip the write if
580
+ the existing file's body already matches the document's content_hash.
581
+
582
+ Collisions in derived paths are resolved by appending a short
583
+ document-id suffix — deterministic across re-runs because it's keyed off
584
+ the immutable UUID, not a counter.
585
+ """
586
+ if _is_directory_unmanaged(vault_path) and not force:
587
+ raise ValueError(
588
+ f"target is not empty and was not created by this tool: "
589
+ f"{vault_path} — pass --force to write into it"
590
+ )
591
+
592
+ init_vault(vault_path)
593
+
594
+ summary = ExportSummary()
595
+ used_paths: set[str] = set()
596
+
597
+ for doc in _iter_documents(conn):
598
+ relative = _resolve_relative_path(doc, used_paths)
599
+ used_paths.add(relative)
600
+ try:
601
+ _, written = _write_doc_file(
602
+ doc, vault_path=vault_path, relative=relative
603
+ )
604
+ except OSError as e:
605
+ summary.errors.append(f"{relative}: {e}")
606
+ continue
607
+ if written:
608
+ summary.written += 1
609
+ else:
610
+ summary.skipped += 1
611
+
612
+ return summary
613
+
614
+
615
+ def regenerate_vault_file(
616
+ conn: psycopg.Connection[Any],
617
+ document_id: str,
618
+ *,
619
+ vault_path: Path,
620
+ force: bool = False,
621
+ ) -> Path:
622
+ """Re-create a single doc's vault mirror file from its DB row.
623
+
624
+ Returns the absolute path of the file written (or, on the idempotent
625
+ skip path, the path that already matched). The body-hash check from
626
+ :func:`export_vault` is preserved by default: if the on-disk body already
627
+ matches ``documents.content_hash``, the file is left untouched.
628
+
629
+ Pass ``force=True`` to bypass the body-hash skip and unconditionally
630
+ rewrite the file. This is the right choice when the caller has already
631
+ determined a change happened (e.g., a frontmatter-only edit such as a
632
+ title, tags, metadata, or content_type update) — the body-hash check
633
+ only fingerprints the body, so it would silently drop the rewrite and
634
+ leave stale frontmatter on disk. The ingest pipeline always sets
635
+ ``force=True`` because it gates the call on a confirmed change
636
+ (created row or frontmatter-bearing field changed). The full-corpus
637
+ :func:`export_vault` keeps ``force=False`` so re-runs stay cheap.
638
+
639
+ Path resolution prefers the doc's existing ``vault_path`` when set —
640
+ a previously-synced ingested doc must regenerate at the same place
641
+ rather than at a freshly computed ``_ingested/<source>/...`` location
642
+ (which would orphan the original mirror). Docs with no ``vault_path``
643
+ fall back to the corpus-dump rules (``_resolve_relative_path``) with
644
+ a fresh, single-doc collision set.
645
+
646
+ Side effect: also UPDATEs ``documents.vault_path`` to the chosen
647
+ relative path so subsequent ``regenerate_vault_file``, ``brain rm``,
648
+ and ``brain tag`` calls all use the same on-disk location. The
649
+ UPDATE is its own statement under ``conn.autocommit=True`` (which
650
+ all current callers use) — a future caller wrapping this function
651
+ in a read-only transaction would have to opt that statement in.
652
+ The ``IS DISTINCT FROM`` predicate keeps it idempotent (NULL-safe):
653
+ re-running on a row that already has the right ``vault_path`` is a
654
+ no-op at the row level.
655
+
656
+ Raises:
657
+ ValueError: ``document_id`` does not match any row.
658
+ ValueError: the row is ``kind='vault'``. Vault-tier authored notes
659
+ are file-source-of-truth — regenerating from the DB risks
660
+ losing edits that haven't been re-synced. Restore from backup
661
+ or git instead.
662
+ OSError: the write itself fails (permissions, disk full, etc.).
663
+ """
664
+ doc = _fetch_document_for_export(conn, document_id)
665
+ if doc is None:
666
+ raise ValueError(f"no document with id {document_id}")
667
+ if doc.kind == "vault":
668
+ raise ValueError(
669
+ "cannot regenerate vault-tier authored note from DB; "
670
+ "restore from backup or git instead"
671
+ )
672
+
673
+ relative = doc.vault_path or _resolve_relative_path(doc, used_paths=set())
674
+
675
+ target, _ = _write_doc_file(
676
+ doc, vault_path=vault_path, relative=relative, force=force
677
+ )
678
+ conn.execute(
679
+ "UPDATE documents SET vault_path = %s "
680
+ "WHERE id = %s AND vault_path IS DISTINCT FROM %s",
681
+ (relative, document_id, relative),
682
+ )
683
+ return target.resolve()