secondbrain-py 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. brain/__init__.py +0 -0
  2. brain/__main__.py +18 -0
  3. brain/_capture_command.py +445 -0
  4. brain/_compose.py +52 -0
  5. brain/activity.py +206 -0
  6. brain/ask.py +631 -0
  7. brain/audio.py +591 -0
  8. brain/backfill/__init__.py +12 -0
  9. brain/backfill/search_extras.py +141 -0
  10. brain/backfill/source_rows.py +101 -0
  11. brain/bin/__init__.py +1 -0
  12. brain/bin/_launcher.py +107 -0
  13. brain/bin/down.py +8 -0
  14. brain/bin/launchd.py +268 -0
  15. brain/bin/monitor.py +570 -0
  16. brain/bin/rebuild.py +8 -0
  17. brain/bin/status.py +8 -0
  18. brain/bin/up.py +8 -0
  19. brain/brief.py +272 -0
  20. brain/capture.py +49 -0
  21. brain/chat.py +293 -0
  22. brain/cli.py +9760 -0
  23. brain/cli_claude.py +81 -0
  24. brain/cli_connect.py +285 -0
  25. brain/cli_demo.py +266 -0
  26. brain/config.py +1949 -0
  27. brain/connect.py +925 -0
  28. brain/db.py +540 -0
  29. brain/demo/__init__.py +452 -0
  30. brain/demo/corpus/manifest.json +403 -0
  31. brain/demo/embedder.py +74 -0
  32. brain/durations.py +84 -0
  33. brain/edit_session.py +156 -0
  34. brain/editor.py +67 -0
  35. brain/elicit/__init__.py +16 -0
  36. brain/elicit/detectors.py +250 -0
  37. brain/elicit/drafter.py +70 -0
  38. brain/elicit/queue.py +220 -0
  39. brain/elicit/schema.py +48 -0
  40. brain/elicit/session.py +445 -0
  41. brain/embedding_targets.py +54 -0
  42. brain/embeddings.py +424 -0
  43. brain/enrichment.py +808 -0
  44. brain/errors.py +357 -0
  45. brain/eval/__init__.py +129 -0
  46. brain/eval/answer_eval.py +281 -0
  47. brain/eval/baseline.py +265 -0
  48. brain/eval/concept_extraction.py +378 -0
  49. brain/eval/corpus.py +152 -0
  50. brain/eval/errors.py +19 -0
  51. brain/eval/graph_baseline.py +226 -0
  52. brain/eval/graph_retrieval.py +202 -0
  53. brain/eval/graph_runner.py +319 -0
  54. brain/eval/metrics.py +101 -0
  55. brain/eval/runner.py +223 -0
  56. brain/format.py +783 -0
  57. brain/gaps.py +390 -0
  58. brain/graph_rag/__init__.py +94 -0
  59. brain/graph_rag/_retrieval_common.py +113 -0
  60. brain/graph_rag/aggregates.py +303 -0
  61. brain/graph_rag/aliases/__init__.py +583 -0
  62. brain/graph_rag/backends/__init__.py +10 -0
  63. brain/graph_rag/backends/_age_helpers.py +473 -0
  64. brain/graph_rag/backends/age.py +782 -0
  65. brain/graph_rag/backends/base.py +272 -0
  66. brain/graph_rag/build.py +344 -0
  67. brain/graph_rag/communities.py +644 -0
  68. brain/graph_rag/communities_summary.py +437 -0
  69. brain/graph_rag/concepts.py +202 -0
  70. brain/graph_rag/cooccur.py +193 -0
  71. brain/graph_rag/cross_type.py +312 -0
  72. brain/graph_rag/extract.py +885 -0
  73. brain/graph_rag/fuse.py +371 -0
  74. brain/graph_rag/global_.py +412 -0
  75. brain/graph_rag/grouping.py +372 -0
  76. brain/graph_rag/person_resolver.py +167 -0
  77. brain/graph_rag/reconcile.py +792 -0
  78. brain/graph_rag/relational.py +353 -0
  79. brain/graph_rag/retrieve.py +526 -0
  80. brain/graph_rag/router.py +288 -0
  81. brain/graph_rag/schema.py +320 -0
  82. brain/graph_rag/sync.py +237 -0
  83. brain/graph_rag/tenancy.py +43 -0
  84. brain/graph_rag/themes.py +501 -0
  85. brain/graph_rag/weighting.py +202 -0
  86. brain/ingest/__init__.py +1926 -0
  87. brain/ingest/chunker.py +249 -0
  88. brain/ingest/docx.py +40 -0
  89. brain/ingest/gmail.py +621 -0
  90. brain/ingest/markdown.py +37 -0
  91. brain/ingest/pdf.py +61 -0
  92. brain/ingest/stdin.py +22 -0
  93. brain/ingest/sub_tokens.py +91 -0
  94. brain/ingest/text.py +16 -0
  95. brain/interactions.py +205 -0
  96. brain/maintenance.py +355 -0
  97. brain/mcp_server.py +3405 -0
  98. brain/migrations/001_init.sql +43 -0
  99. brain/migrations/002_qwen3_embedding.sql +17 -0
  100. brain/migrations/003_vault_model.sql +41 -0
  101. brain/migrations/004_relax_content_hash_uniqueness.sql +18 -0
  102. brain/migrations/005_derived_links.sql +67 -0
  103. brain/migrations/006_dedup_file_by_source_path.sql +25 -0
  104. brain/migrations/007_email_thread_and_draft.sql +15 -0
  105. brain/migrations/008_gmail_thread_unique.sql +11 -0
  106. brain/migrations/009_chunks_weighted_tsv.sql +28 -0
  107. brain/migrations/010_interactions.sql +30 -0
  108. brain/migrations/011_documents_summary.sql +23 -0
  109. brain/migrations/012_graphrag.sql +171 -0
  110. brain/migrations/013_graphrag_communities.sql +125 -0
  111. brain/migrations/014_graphrag_community_summary_hash.sql +33 -0
  112. brain/migrations/015_interactions_graph_targets.sql +89 -0
  113. brain/migrations/016_index_hygiene.sql +61 -0
  114. brain/migrations/017_elicit.sql +30 -0
  115. brain/migrations/018_review_gap_signal_kinds.sql +40 -0
  116. brain/migrations/019_search_queries.sql +35 -0
  117. brain/migrations/020_link_suggestions.sql +40 -0
  118. brain/migrations/021_timeline_doc_date.sql +34 -0
  119. brain/migrations/022_link_suggestions_undirected.sql +84 -0
  120. brain/migrations/023_search_queries_fts_count.sql +28 -0
  121. brain/quartz_overrides/__init__.py +8 -0
  122. brain/quartz_overrides/quartz/bootstrap-cli.mjs +65 -0
  123. brain/quartz_overrides/quartz/build.ts +568 -0
  124. brain/quartz_overrides/quartz/cli/args.js +152 -0
  125. brain/quartz_overrides/quartz/cli/build_partial_handler.js +544 -0
  126. brain/quartz_overrides/quartz/cli/handlers.js +636 -0
  127. brain/quartz_overrides/quartz/components/CommandPalette.tsx +172 -0
  128. brain/quartz_overrides/quartz/components/Explorer.tsx +198 -0
  129. brain/quartz_overrides/quartz/components/Footer.tsx +27 -0
  130. brain/quartz_overrides/quartz/components/Graph.tsx +468 -0
  131. brain/quartz_overrides/quartz/components/PageTitle.tsx +72 -0
  132. brain/quartz_overrides/quartz/components/RelatedDocs.tsx +38 -0
  133. brain/quartz_overrides/quartz/components/Search.tsx +161 -0
  134. brain/quartz_overrides/quartz/components/SummaryLede.tsx +72 -0
  135. brain/quartz_overrides/quartz/components/index.ts +92 -0
  136. brain/quartz_overrides/quartz/components/pages/TagContent.tsx +272 -0
  137. brain/quartz_overrides/quartz/components/scripts/commandPalette.inline.ts +665 -0
  138. brain/quartz_overrides/quartz/components/scripts/explorer.inline.ts +768 -0
  139. brain/quartz_overrides/quartz/components/scripts/graph.inline.ts +2302 -0
  140. brain/quartz_overrides/quartz/components/scripts/relatedDocs.inline.ts +163 -0
  141. brain/quartz_overrides/quartz/components/scripts/search.inline.ts +1011 -0
  142. brain/quartz_overrides/quartz/plugins/emitters/contentIndex.ts +546 -0
  143. brain/quartz_overrides/quartz/plugins/transformers/codeCopy.ts +94 -0
  144. brain/quartz_overrides/quartz/plugins/transformers/derivedFenceMark.ts +302 -0
  145. brain/quartz_overrides/quartz/plugins/transformers/emailThread.ts +148 -0
  146. brain/quartz_overrides/quartz/plugins/transformers/emptyDoorFilter.ts +213 -0
  147. brain/quartz_overrides/quartz/plugins/transformers/index.ts +114 -0
  148. brain/quartz_overrides/quartz/plugins/transformers/linkKindMark.ts +205 -0
  149. brain/quartz_overrides/quartz/plugins/transformers/linkSourceTag.ts +104 -0
  150. brain/quartz_overrides/quartz/plugins/transformers/relativeDate.ts +100 -0
  151. brain/quartz_overrides/quartz/plugins/transformers/reloadSignal.ts +131 -0
  152. brain/quartz_overrides/quartz/processors/parse.ts +371 -0
  153. brain/quartz_overrides/quartz/processors/parser_cache.ts +78 -0
  154. brain/quartz_overrides/quartz/static/brain-logo-dark.png +0 -0
  155. brain/quartz_overrides/quartz/static/brain-logo-light.png +0 -0
  156. brain/quartz_overrides/quartz/static/codeCopy.js +196 -0
  157. brain/quartz_overrides/quartz/static/emailThread.js +334 -0
  158. brain/quartz_overrides/quartz/static/favicon.ico +0 -0
  159. brain/quartz_overrides/quartz/static/icon.png +0 -0
  160. brain/quartz_overrides/quartz/static/linkSourceTag.js +104 -0
  161. brain/quartz_overrides/quartz/static/relativeDate.js +142 -0
  162. brain/quartz_overrides/quartz/static/reload.js +168 -0
  163. brain/quartz_overrides/quartz/styles/brain/_article.scss +252 -0
  164. brain/quartz_overrides/quartz/styles/brain/_atmosphere.scss +113 -0
  165. brain/quartz_overrides/quartz/styles/brain/_callouts.scss +180 -0
  166. brain/quartz_overrides/quartz/styles/brain/_cmdk.scss +7 -0
  167. brain/quartz_overrides/quartz/styles/brain/_code.scss +208 -0
  168. brain/quartz_overrides/quartz/styles/brain/_command_palette.scss +369 -0
  169. brain/quartz_overrides/quartz/styles/brain/_email_thread.scss +228 -0
  170. brain/quartz_overrides/quartz/styles/brain/_explorer.scss +142 -0
  171. brain/quartz_overrides/quartz/styles/brain/_home.scss +182 -0
  172. brain/quartz_overrides/quartz/styles/brain/_links.scss +322 -0
  173. brain/quartz_overrides/quartz/styles/brain/_marginalia.scss +117 -0
  174. brain/quartz_overrides/quartz/styles/brain/_motion.scss +175 -0
  175. brain/quartz_overrides/quartz/styles/brain/_people_hub.scss +100 -0
  176. brain/quartz_overrides/quartz/styles/brain/_related_docs.scss +137 -0
  177. brain/quartz_overrides/quartz/styles/brain/_search.scss +252 -0
  178. brain/quartz_overrides/quartz/styles/brain/_sidebar.scss +468 -0
  179. brain/quartz_overrides/quartz/styles/brain/_summary_lede.scss +56 -0
  180. brain/quartz_overrides/quartz/styles/brain/_surface.scss +43 -0
  181. brain/quartz_overrides/quartz/styles/brain/_tag_content.scss +118 -0
  182. brain/quartz_overrides/quartz/styles/brain/_tokens.scss +197 -0
  183. brain/quartz_overrides/quartz/styles/brain/_typography.scss +92 -0
  184. brain/quartz_overrides/quartz/styles/custom.scss +89 -0
  185. brain/quartz_overrides/quartz/styles/graph.scss +505 -0
  186. brain/quartz_overrides/quartz/util/ctx.ts +92 -0
  187. brain/quartz_overrides/quartz/util/fastpath_manifest.ts +608 -0
  188. brain/quartz_overrides/quartz/util/path.ts +358 -0
  189. brain/quartz_overrides/quartz/util/sourceIcons.ts +55 -0
  190. brain/quartz_overrides/quartz.config.ts +270 -0
  191. brain/quartz_overrides/quartz.layout.ts +314 -0
  192. brain/queries.py +1188 -0
  193. brain/rank_fusion.py +8 -0
  194. brain/resurface.py +210 -0
  195. brain/review/__init__.py +26 -0
  196. brain/review/emit.py +27 -0
  197. brain/review/queries.py +436 -0
  198. brain/review/render.py +196 -0
  199. brain/review/scans.py +355 -0
  200. brain/review/weekly.py +413 -0
  201. brain/search.py +704 -0
  202. brain/set_similarity.py +15 -0
  203. brain/setup.py +1205 -0
  204. brain/tags.py +56 -0
  205. brain/templates/Caddyfile.j2 +9 -0
  206. brain/templates/__init__.py +1 -0
  207. brain/templates/bin/__init__.py +1 -0
  208. brain/templates/bin/_brain-brief-fg.sh +25 -0
  209. brain/templates/bin/_brain-build-fg.sh +53 -0
  210. brain/templates/bin/_brain-watcher-fg.sh +65 -0
  211. brain/templates/bin/brain-down.sh +89 -0
  212. brain/templates/bin/brain-status.sh +83 -0
  213. brain/templates/bin/brain-up.sh +221 -0
  214. brain/templates/docker/age/Dockerfile +79 -0
  215. brain/templates/docker-compose.stock.yml.j2 +26 -0
  216. brain/templates/docker-compose.yml.j2 +34 -0
  217. brain/templates/env.example +190 -0
  218. brain/templates/launchd/__init__.py +1 -0
  219. brain/templates/launchd/com.brain.brief.plist.j2 +45 -0
  220. brain/templates/launchd/com.brain.build.plist.j2 +46 -0
  221. brain/templates/launchd/com.brain.watcher.plist.j2 +46 -0
  222. brain/templates/skill/SKILL.md +63 -0
  223. brain/templates/skill/__init__.py +1 -0
  224. brain/timeline.py +834 -0
  225. brain/todo.py +124 -0
  226. brain/uninstall.py +185 -0
  227. brain/vault/__init__.py +115 -0
  228. brain/vault/_atomic.py +25 -0
  229. brain/vault/daily_index.py +228 -0
  230. brain/vault/derived_links/__init__.py +50 -0
  231. brain/vault/derived_links/directory.py +683 -0
  232. brain/vault/derived_links/fence.py +408 -0
  233. brain/vault/derived_links/gws.py +64 -0
  234. brain/vault/derived_links/participants.py +143 -0
  235. brain/vault/derived_links/pass_runner.py +362 -0
  236. brain/vault/derived_links/rules.py +137 -0
  237. brain/vault/export.py +683 -0
  238. brain/vault/frontmatter.py +165 -0
  239. brain/vault/graph.py +620 -0
  240. brain/vault/graph_format.py +388 -0
  241. brain/vault/link_rewrite.py +235 -0
  242. brain/vault/links.py +260 -0
  243. brain/vault/note_builder.py +211 -0
  244. brain/vault/paths.py +55 -0
  245. brain/vault/quartz_overlay.py +236 -0
  246. brain/vault/rename.py +591 -0
  247. brain/vault/resolver.py +304 -0
  248. brain/vault/slug.py +127 -0
  249. brain/vault/sync.py +1513 -0
  250. brain/vault/sync_summaries.py +264 -0
  251. brain/vault/templates.py +145 -0
  252. brain/vault/watch.py +1052 -0
  253. brain/wiki/__init__.py +6 -0
  254. brain/wiki/_github_slugger.py +76 -0
  255. brain/wiki/_person_name.py +314 -0
  256. brain/wiki/build_homepage.py +541 -0
  257. brain/wiki/build_partial.py +273 -0
  258. brain/wiki/build_people.py +934 -0
  259. brain/wiki/build_related.py +758 -0
  260. brain/wiki/build_swap.py +585 -0
  261. brain/wiki/build_watcher.py +975 -0
  262. brain/wiki/edit_classifier.py +215 -0
  263. brain/wiki/errors.py +10 -0
  264. brain/wiki/fastpath_manifest.py +475 -0
  265. brain/wiki/fastpath_state.py +174 -0
  266. brain/wiki/install.py +296 -0
  267. brain/wiki/slug.py +111 -0
  268. secondbrain_py-0.2.1.dist-info/METADATA +195 -0
  269. secondbrain_py-0.2.1.dist-info/RECORD +273 -0
  270. secondbrain_py-0.2.1.dist-info/WHEEL +5 -0
  271. secondbrain_py-0.2.1.dist-info/entry_points.txt +11 -0
  272. secondbrain_py-0.2.1.dist-info/licenses/LICENSE +21 -0
  273. secondbrain_py-0.2.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,371 @@
1
+ // Brain wiki — parse pipeline with per-file parser result cache.
2
+ //
3
+ // This file is a TEMPLATE. It is installed at
4
+ // `<vault>/.quartz/quartz/processors/parse.ts` by
5
+ // `brain vault render --overlay`, overwriting the stock upstream module.
6
+ //
7
+ // Changes from upstream:
8
+ // 1. `createFileParser` loop: reads raw bytes with `readFile` (replaces
9
+ // `read` from to-vfile — equivalent but gives us the Buffer for hashing).
10
+ // On each file, computes a cache key and checks the MDAST cache before
11
+ // running `processor.parse` + `processor.run`. Hit → push reconstituted
12
+ // MarkdownContent and continue. Miss → run existing path, then store.
13
+ // Only the MDAST phase is cached: the cache key is deterministic on
14
+ // (file_bytes, slug) and the dominant build cost is `processor.parse` +
15
+ // `processor.run` (remark), not the remark-rehype step that follows.
16
+ // The HTML phase (`createMarkdownParser`) is left identical to upstream.
17
+ // 2. `parseMarkdown`: computes `cacheDir` (defaults to
18
+ // `<argv.directory>/.quartz/.cache/parser` unless ctx.cacheDir is already
19
+ // set or `argv.noCache` is true). Propagates `cacheDir` to workers via
20
+ // the `serializableCtx` literal.
21
+ //
22
+ // Cache module: `./parser_cache.ts` — pure Node.js, no Quartz-internal
23
+ // imports, tested independently in `tests/test_quartz_parser_cache.ts`.
24
+ //
25
+ // Safety: every MDAST transformer in quartz_overrides/quartz/plugins/
26
+ // transformers/ is a pure function of (file bytes, slug) — no cross-file
27
+ // reads at the parse stage. Cross-file work (backlinks, contentIndex,
28
+ // related-docs) happens in the emitter phase on the in-memory
29
+ // ProcessedContent[] collection AFTER this function returns. The
30
+ // transformers/index.ts file documents this contract.
31
+ //
32
+ // Tested against Quartz v4.5.x (April 2026). If a future Quartz version
33
+ // changes the parse pipeline shape (e.g. adds a third processor phase or
34
+ // changes how VFile is constructed), diff upstream against this file and
35
+ // re-apply the brain-cache hooks.
36
+
37
+ import esbuild from "esbuild"
38
+ import remarkParse from "remark-parse"
39
+ import remarkRehype from "remark-rehype"
40
+ import { Processor, unified } from "unified"
41
+ import { Root as MDRoot } from "remark-parse/lib"
42
+ import { Root as HTMLRoot } from "hast"
43
+ import { MarkdownContent, ProcessedContent } from "../plugins/vfile"
44
+ import { PerfTimer } from "../util/perf"
45
+ import { readFile } from "node:fs/promises"
46
+ import { VFile } from "vfile"
47
+ import { FilePath, QUARTZ, slugifyFilePath } from "../util/path"
48
+ import path from "path"
49
+ import workerpool, { Promise as WorkerPromise } from "workerpool"
50
+ import { QuartzLogger } from "../util/log"
51
+ import { trace } from "../util/trace"
52
+ import { BuildCtx, WorkerSerializableBuildCtx } from "../util/ctx"
53
+ import { styleText } from "util"
54
+ import { cacheKey, getCached, putCached, CACHE_VERSION } from "./parser_cache"
55
+
56
+ export type QuartzMdProcessor = Processor<MDRoot, MDRoot, MDRoot>
57
+ export type QuartzHtmlProcessor = Processor<undefined, MDRoot, HTMLRoot>
58
+
59
+ export function createMdProcessor(ctx: BuildCtx): QuartzMdProcessor {
60
+ const transformers = ctx.cfg.plugins.transformers
61
+
62
+ return (
63
+ unified()
64
+ // base Markdown -> MD AST
65
+ .use(remarkParse)
66
+ // MD AST -> MD AST transforms
67
+ .use(
68
+ transformers.flatMap((plugin) => plugin.markdownPlugins?.(ctx) ?? []),
69
+ ) as unknown as QuartzMdProcessor
70
+ // ^ sadly the typing of `use` is not smart enough to infer the correct type from our plugin list
71
+ )
72
+ }
73
+
74
+ export function createHtmlProcessor(ctx: BuildCtx): QuartzHtmlProcessor {
75
+ const transformers = ctx.cfg.plugins.transformers
76
+ return (
77
+ unified()
78
+ // MD AST -> HTML AST
79
+ .use(remarkRehype, { allowDangerousHtml: true })
80
+ // HTML AST -> HTML AST transforms
81
+ .use(transformers.flatMap((plugin) => plugin.htmlPlugins?.(ctx) ?? []))
82
+ )
83
+ }
84
+
85
+ function* chunks<T>(arr: T[], n: number) {
86
+ for (let i = 0; i < arr.length; i += n) {
87
+ yield arr.slice(i, i + n)
88
+ }
89
+ }
90
+
91
+ // brain-cache: rehydrate Date instances inside cached file.data.
92
+ // JSON.stringify turns Date objects into ISO strings; on read they come
93
+ // back as strings, and components that call `.toISOString()` on them
94
+ // crash (`date.toISOString is not a function`). The upstream `lastmod`
95
+ // transformer populates `data.dates = { created, modified, published }`
96
+ // with Dates; this helper restores them in place. Keep this list in sync
97
+ // with `quartz/plugins/vfile.ts` `DataMap.dates` if upstream adds fields.
98
+ function rehydrateDates(data: Record<string, unknown>): void {
99
+ const dates = data.dates as Record<string, unknown> | undefined
100
+ if (!dates || typeof dates !== "object") return
101
+ for (const key of ["created", "modified", "published"] as const) {
102
+ const value = dates[key]
103
+ if (typeof value === "string") {
104
+ dates[key] = new Date(value)
105
+ }
106
+ }
107
+ }
108
+
109
+ async function transpileWorkerScript() {
110
+ // transpile worker script
111
+ const cacheFile = "./.quartz-cache/transpiled-worker.mjs"
112
+ const fp = "./quartz/worker.ts"
113
+ return esbuild.build({
114
+ entryPoints: [fp],
115
+ outfile: path.join(QUARTZ, cacheFile),
116
+ bundle: true,
117
+ keepNames: true,
118
+ platform: "node",
119
+ format: "esm",
120
+ packages: "external",
121
+ sourcemap: true,
122
+ sourcesContent: false,
123
+ plugins: [
124
+ {
125
+ name: "css-and-scripts-as-text",
126
+ setup(build) {
127
+ build.onLoad({ filter: /\.scss$/ }, (_) => ({
128
+ contents: "",
129
+ loader: "text",
130
+ }))
131
+ build.onLoad({ filter: /\.inline\.(ts|js)$/ }, (_) => ({
132
+ contents: "",
133
+ loader: "text",
134
+ }))
135
+ },
136
+ },
137
+ ],
138
+ })
139
+ }
140
+
141
+ // brain-cache: shape of a stored MDAST cache entry.
142
+ type MdCacheEntry = {
143
+ version: number
144
+ slug: string
145
+ ast: MDRoot
146
+ data: Record<string, unknown>
147
+ }
148
+
149
+ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
150
+ const { argv, cfg } = ctx
151
+ return async (processor: QuartzMdProcessor) => {
152
+ const res: MarkdownContent[] = []
153
+ for (const fp of fps) {
154
+ try {
155
+ const perf = new PerfTimer()
156
+
157
+ // brain-cache: read raw bytes once; re-use as VFile value and as
158
+ // hash input. Equivalent to `await read(fp)` from to-vfile but
159
+ // gives us a Buffer for the cache key without a second fs.readFile.
160
+ const rawBytes = await readFile(fp)
161
+ const file = new VFile({ path: fp as string, value: rawBytes })
162
+
163
+ // strip leading and trailing whitespace
164
+ file.value = file.value.toString().trim()
165
+
166
+ // Text -> Text transforms
167
+ for (const plugin of cfg.plugins.transformers.filter((p) => p.textTransform)) {
168
+ file.value = plugin.textTransform!(ctx, file.value.toString())
169
+ }
170
+
171
+ // base data properties that plugins may use
172
+ file.data.filePath = file.path as FilePath
173
+ file.data.relativePath = path.posix.relative(argv.directory, file.path!) as FilePath
174
+ file.data.slug = slugifyFilePath(file.data.relativePath as FilePath)
175
+
176
+ // brain-fastpath: snapshot raw source BEFORE OFM transformer mutates vfile.value.
177
+ // computeFingerprint (fastpath_manifest.ts) reads vfile.data.rawSource to get the
178
+ // unmodified markdown text for canonical-blob building. The OFM transformer runs
179
+ // later in processor.run — e.g. it appends |^blockId display aliases to transclusion
180
+ // links (![[target#^block]] → ![[target#^block|^block]]). We store the raw disk
181
+ // bytes (rawBytes is the original Buffer from fs.readFile) so the snapshot is
182
+ // byte-identical to what parse.ts originally loaded, regardless of any later
183
+ // textTransform / trim / OFM mutations.
184
+ ;(file.data as Record<string, unknown>)["rawSource"] = rawBytes.toString("utf8")
185
+
186
+ // brain-cache: check MDAST cache before running the expensive parse.
187
+ const mdKey = ctx.cacheDir ? cacheKey(rawBytes, file.data.slug as string) : null
188
+ if (mdKey !== null) {
189
+ const cached = getCached<MdCacheEntry>(ctx.cacheDir!, mdKey)
190
+ // belt-and-suspenders: version is already baked into the cache key (the
191
+ // key hash includes CACHE_VERSION), so a stale entry at a different key
192
+ // path is simply a miss. The explicit version check here guards the
193
+ // unlikely edge case where two builds with different CACHE_VERSION values
194
+ // happen to collide on a key (sha256 collision) — effectively impossible,
195
+ // but cheap to double-check.
196
+ if (cached !== null && cached.version === CACHE_VERSION) {
197
+ // Reconstitute MarkdownContent from cached entry.
198
+ // value is set to the trimmed string so cache-hit and cache-miss
199
+ // paths produce the same file.value shape for downstream emitters.
200
+ const cachedFile = new VFile({ path: fp as string, value: rawBytes })
201
+ cachedFile.value = rawBytes.toString().trim()
202
+ // Rehydrate Date instances that JSON serialization stringified.
203
+ // Mutates cached.data in place — fine, the cache file on disk is
204
+ // not re-read and the entry won't be written back.
205
+ rehydrateDates(cached.data as Record<string, unknown>)
206
+ cachedFile.data = cached.data as typeof file.data
207
+ // brain-fastpath: always inject rawSource from current rawBytes even on cache hit.
208
+ // Old cache entries written before this snapshot was introduced may lack rawSource;
209
+ // setting it here ensures computeFingerprint never encounters a missing field
210
+ // without requiring a CACHE_VERSION bump.
211
+ ;(cachedFile.data as Record<string, unknown>)["rawSource"] = rawBytes.toString("utf8")
212
+ res.push([cached.ast, cachedFile])
213
+ if (argv.verbose) {
214
+ console.log(`[markdown hit] ${fp} -> ${cachedFile.data.slug} (${perf.timeSince()})`)
215
+ }
216
+ continue
217
+ }
218
+ }
219
+
220
+ const ast = processor.parse(file)
221
+ const newAst = await processor.run(ast, file)
222
+
223
+ // brain-cache: push BEFORE putCached so the file is always emitted
224
+ // even if putCached throws (ENOSPC / EACCES / readonly FS). A write
225
+ // failure must never silently drop a file from the build output.
226
+ res.push([newAst, file])
227
+
228
+ // brain-cache: store result for next build.
229
+ if (mdKey !== null) {
230
+ putCached<MdCacheEntry>(ctx.cacheDir!, mdKey, {
231
+ version: CACHE_VERSION,
232
+ slug: file.data.slug as string,
233
+ ast: newAst,
234
+ data: file.data as Record<string, unknown>,
235
+ })
236
+ }
237
+
238
+ if (argv.verbose) {
239
+ console.log(`[markdown] ${fp} -> ${file.data.slug} (${perf.timeSince()})`)
240
+ }
241
+ } catch (err) {
242
+ trace(`\nFailed to process markdown \`${fp}\``, err as Error)
243
+ }
244
+ }
245
+
246
+ return res
247
+ }
248
+ }
249
+
250
+ export function createMarkdownParser(ctx: BuildCtx, mdContent: MarkdownContent[]) {
251
+ return async (processor: QuartzHtmlProcessor) => {
252
+ const res: ProcessedContent[] = []
253
+ for (const [ast, file] of mdContent) {
254
+ try {
255
+ const perf = new PerfTimer()
256
+
257
+ const newAst = await processor.run(ast as MDRoot, file)
258
+ res.push([newAst, file])
259
+
260
+ if (ctx.argv.verbose) {
261
+ console.log(`[html] ${file.data.slug} (${perf.timeSince()})`)
262
+ }
263
+ } catch (err) {
264
+ trace(`\nFailed to process html \`${file.data.filePath}\``, err as Error)
265
+ }
266
+ }
267
+
268
+ return res
269
+ }
270
+ }
271
+
272
+ const clamp = (num: number, min: number, max: number) =>
273
+ Math.min(Math.max(Math.round(num), min), max)
274
+
275
+ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<ProcessedContent[]> {
276
+ const { argv } = ctx
277
+ const perf = new PerfTimer()
278
+ const log = new QuartzLogger(argv.verbose)
279
+
280
+ // brain-cache: resolve cacheDir. Precedence:
281
+ // 1. ctx.cacheDir already set by caller (string | null | undefined)
282
+ // 2. argv.noCache === true → null (disable for this run)
283
+ // 3. default → <argv.directory>/.quartz/.cache/parser
284
+ // `cacheDir: undefined` from an upstream BuildCtx constructor means
285
+ // "not set yet" and falls through to the default.
286
+ // NOTE: this path must match brain.maintenance._PARSER_CACHE_RELPATH
287
+ // (.quartz/.cache/parser), wiped by `brain-rebuild --clean-cache`.
288
+ const cacheDir: string | null =
289
+ ctx.cacheDir !== undefined
290
+ ? ctx.cacheDir
291
+ : argv.noCache
292
+ ? null
293
+ : path.join(argv.directory, ".quartz", ".cache", "parser")
294
+
295
+ const patchedCtx: BuildCtx = { ...ctx, cacheDir }
296
+
297
+ // rough heuristics: 128 gives enough time for v8 to JIT and optimize parsing code paths
298
+ const CHUNK_SIZE = 128
299
+ const concurrency = argv.concurrency ?? clamp(fps.length / CHUNK_SIZE, 1, 4)
300
+
301
+ let res: ProcessedContent[] = []
302
+ log.start(`Parsing input files using ${concurrency} threads`)
303
+ if (concurrency === 1) {
304
+ try {
305
+ const mdRes = await createFileParser(patchedCtx, fps)(createMdProcessor(patchedCtx))
306
+ res = await createMarkdownParser(patchedCtx, mdRes)(createHtmlProcessor(patchedCtx))
307
+ } catch (error) {
308
+ log.end()
309
+ throw error
310
+ }
311
+ } else {
312
+ await transpileWorkerScript()
313
+ const pool = workerpool.pool("./quartz/bootstrap-worker.mjs", {
314
+ minWorkers: "max",
315
+ maxWorkers: concurrency,
316
+ workerType: "thread",
317
+ })
318
+ const errorHandler = (err: any) => {
319
+ console.error(err)
320
+ process.exit(1)
321
+ }
322
+
323
+ // brain-cache: include cacheDir so workers can read/write the same
324
+ // on-disk cache. WorkerSerializableBuildCtx = Omit<BuildCtx, "cfg" | "trie">
325
+ // so cacheDir is automatically part of the type; this literal is explicit
326
+ // per the upstream pattern (workers deserialise by spreading partialCtx).
327
+ const serializableCtx: WorkerSerializableBuildCtx = {
328
+ buildId: patchedCtx.buildId,
329
+ argv: patchedCtx.argv,
330
+ allSlugs: patchedCtx.allSlugs,
331
+ allFiles: patchedCtx.allFiles,
332
+ incremental: patchedCtx.incremental,
333
+ cacheDir: patchedCtx.cacheDir,
334
+ }
335
+
336
+ const textToMarkdownPromises: WorkerPromise<MarkdownContent[]>[] = []
337
+ let processedFiles = 0
338
+ for (const chunk of chunks(fps, CHUNK_SIZE)) {
339
+ textToMarkdownPromises.push(pool.exec("parseMarkdown", [serializableCtx, chunk]))
340
+ }
341
+
342
+ const mdResults: Array<MarkdownContent[]> = await Promise.all(
343
+ textToMarkdownPromises.map(async (promise) => {
344
+ const result = await promise
345
+ processedFiles += result.length
346
+ log.updateText(`text->markdown ${styleText("gray", `${processedFiles}/${fps.length}`)}`)
347
+ return result
348
+ }),
349
+ ).catch(errorHandler)
350
+
351
+ const markdownToHtmlPromises: WorkerPromise<ProcessedContent[]>[] = []
352
+ processedFiles = 0
353
+ for (const mdChunk of mdResults) {
354
+ markdownToHtmlPromises.push(pool.exec("processHtml", [serializableCtx, mdChunk]))
355
+ }
356
+ const results: ProcessedContent[][] = await Promise.all(
357
+ markdownToHtmlPromises.map(async (promise) => {
358
+ const result = await promise
359
+ processedFiles += result.length
360
+ log.updateText(`markdown->html ${styleText("gray", `${processedFiles}/${fps.length}`)}`)
361
+ return result
362
+ }),
363
+ ).catch(errorHandler)
364
+
365
+ res = results.flat()
366
+ await pool.terminate()
367
+ }
368
+
369
+ log.end(`Parsed ${res.length} Markdown files in ${perf.timeSince()}`)
370
+ return res
371
+ }
@@ -0,0 +1,78 @@
1
+ // Pure file-IO parser result cache for the Quartz incremental-build pipeline.
2
+ //
3
+ // This file is a TEMPLATE. It is installed at
4
+ // `<vault>/.quartz/quartz/processors/parser_cache.ts` by
5
+ // `brain vault render --overlay`. It does NOT compile or run from the
6
+ // brain repo itself; it is pure Node.js with no Quartz-internal imports
7
+ // so it can also be unit-tested directly from the brain repo with
8
+ // `tsx --test tests/test_quartz_parser_cache.ts`.
9
+ //
10
+ // Cache key: sha256(CACHE_VERSION as 4-byte big-endian || slug as UTF-8 || fileBytes).
11
+ // Version baked into key — bumping CACHE_VERSION changes every key and
12
+ // therefore makes every existing entry unreachable without deleting them.
13
+ // Disk: <cacheDir>/<key[0:2]>/<key[2:4]>/<key>.json (two-level sharding
14
+ // keeps any single directory under a few thousand entries at vault scale).
15
+ // Atomicity: write to <path>.<pid>.<uuid>.tmp, then fs.renameSync (POSIX-atomic).
16
+ // pid + randomUUID() guarantees a unique tmp path per write even across
17
+ // worker_threads sharing the same pid, so concurrent writes to the same key
18
+ // are independent: they race to rename identical content and the last rename
19
+ // wins without corruption.
20
+ // Eviction: none. Disk grows O(vault size). The entire cache directory
21
+ // can be deleted at any time — the next build regenerates it.
22
+
23
+ import { createHash, randomUUID } from "node:crypto"
24
+ import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"
25
+ import { dirname, join } from "node:path"
26
+
27
+ export const CACHE_VERSION = 1
28
+
29
+ // cacheKey computes sha256(version-bytes || slug-utf8 || fileBytes) as lowercase hex.
30
+ // CACHE_VERSION is mixed in first so a global version bump changes every key.
31
+ export function cacheKey(fileBytes: Buffer, slug: string): string {
32
+ const hash = createHash("sha256")
33
+ const versionBuf = Buffer.allocUnsafe(4)
34
+ versionBuf.writeUInt32BE(CACHE_VERSION, 0)
35
+ hash.update(versionBuf)
36
+ hash.update(slug, "utf8")
37
+ hash.update(fileBytes)
38
+ return hash.digest("hex")
39
+ }
40
+
41
+ // cachePath returns the on-disk JSON path for a given key under cacheDir.
42
+ // Two-level shard: <aa>/<bb>/<full-key>.json where aa=key[0:2], bb=key[2:4].
43
+ export function cachePath(cacheDir: string, key: string): string {
44
+ return join(cacheDir, key.slice(0, 2), key.slice(2, 4), `${key}.json`)
45
+ }
46
+
47
+ // getCached reads and JSON-parses the cache entry for key.
48
+ // Returns null on ENOENT (clean miss) or SyntaxError (corrupt/truncated file —
49
+ // logged as a warning so the next build silently regenerates the entry without
50
+ // crashing the whole build). Any other unexpected error (permissions, I/O, etc.)
51
+ // is rethrown.
52
+ export function getCached<T>(cacheDir: string, key: string): T | null {
53
+ const p = cachePath(cacheDir, key)
54
+ try {
55
+ const raw = readFileSync(p, "utf8")
56
+ return JSON.parse(raw) as T
57
+ } catch (err: unknown) {
58
+ if (err instanceof Error && "code" in err && (err as NodeJS.ErrnoException).code === "ENOENT") {
59
+ return null
60
+ }
61
+ if (err instanceof SyntaxError) {
62
+ console.warn(`[parser_cache] discarding corrupt entry ${p}: ${err.message}`)
63
+ return null
64
+ }
65
+ throw err
66
+ }
67
+ }
68
+
69
+ // putCached serialises value to JSON and writes it atomically via a tmp file.
70
+ // mkdir -p is called on the shard directory so the first write to a shard
71
+ // creates the directory tree.
72
+ export function putCached<T>(cacheDir: string, key: string, value: T): void {
73
+ const p = cachePath(cacheDir, key)
74
+ mkdirSync(dirname(p), { recursive: true })
75
+ const tmp = `${p}.${process.pid}.${randomUUID()}.tmp`
76
+ writeFileSync(tmp, JSON.stringify(value))
77
+ renameSync(tmp, p)
78
+ }
@@ -0,0 +1,196 @@
1
+ // Brain wiki redesign — code-block copy-button injector (Lane C).
2
+ //
3
+ // This file is a TEMPLATE. It is installed at
4
+ // `<vault>/.quartz/quartz/static/codeCopy.js` by `brain vault render
5
+ // --overlay`, and is copied verbatim into every build's `static/` dir
6
+ // by Quartz's stock `Plugin.Static()` emitter (no extra wiring needed
7
+ // — Quartz already mirrors `quartz/static/` 1:1 into the build
8
+ // output). The script is referenced from page `<head>` by the brain
9
+ // `Plugin.CodeCopy` transformer's `externalResources()` hook (see
10
+ // `codeCopy.ts` for the inject side).
11
+ //
12
+ // Tested against Quartz v4.5.x (April 2026). Plain vanilla JS — no
13
+ // transpile, no bundler — so it survives any future Quartz refactor
14
+ // of its plugin pipeline.
15
+ //
16
+ // What this script does, and why: Quartz's stock `Body.tsx` wires a
17
+ // `.clipboard-button` into every `<pre>`, but its visual treatment
18
+ // (gray border, white background, top-right floating chip) reads as
19
+ // 2018-Hugo. The 2026 redesign wants a small-caps "Copy" pill with
20
+ // the redesign palette, an "↗ Copied" success state, ARIA labelling,
21
+ // and graceful keyboard focus. Rather than rewrite `Body.tsx` (which
22
+ // would require also vendoring its clipboard.inline.ts), we hide
23
+ // the stock button via CSS in `_code.scss` and inject our own
24
+ // `.brain-code-copy` button here.
25
+ //
26
+ // Idempotency contract: every `<pre>` gets at most one
27
+ // `.brain-code-copy` button. We mark each processed `<pre>` with
28
+ // `data-brain-copy-injected="true"` so SPA navigation re-runs of
29
+ // the tagger don't double-inject. First writer wins.
30
+ //
31
+ // Also handles the `data-language` attribute fallback. Quartz's
32
+ // `Plugin.SyntaxHighlighting` stamps `<pre data-language="bash">`
33
+ // directly, but for fenced code blocks that the highlighter doesn't
34
+ // match (e.g. unrecognised languages), the attribute lands only on
35
+ // the inner `<code class="language-X">`. The CSS-only language label
36
+ // in `_code.scss` reads the attribute via `attr()`, which only works
37
+ // on `pre[data-language]`. This script copies the attribute from the
38
+ // inner `<code>` to the outer `<pre>` whenever the outer is missing
39
+ // it, so the label reliably renders.
40
+ //
41
+ // Browser support: targets evergreen Chrome / Firefox / Safari. Uses
42
+ // `navigator.clipboard.writeText`, `document.querySelectorAll`, and
43
+ // `addEventListener` — all baseline since 2017.
44
+ ;(function () {
45
+ // brain: the data-attribute we stamp on processed `<pre>` blocks.
46
+ // Used as the idempotency check.
47
+ var INJECTED_ATTR = "data-brain-copy-injected"
48
+
49
+ // brain: regex matching `language-X` classes Quartz writes onto
50
+ // `<code>` for fenced blocks. Captures the language token.
51
+ var LANGUAGE_CLASS_RE = /(?:^|\s)language-([\w+-]+)/
52
+
53
+ // brain: copy the inner `<code class="language-X">`'s language token
54
+ // up to the outer `<pre data-language="X">` whenever the outer is
55
+ // missing it. CSS reads from the outer; the highlighter sometimes
56
+ // only stamps the inner. Idempotent — skips when outer already has
57
+ // the attribute.
58
+ function liftLanguageAttr(pre) {
59
+ if (pre.hasAttribute("data-language")) return
60
+ var code = pre.querySelector("code")
61
+ if (code === null) return
62
+ var match = LANGUAGE_CLASS_RE.exec(code.className)
63
+ if (match === null) return
64
+ pre.setAttribute("data-language", match[1])
65
+ }
66
+
67
+ // brain: extract the code text the user wants to copy. Stock
68
+ // Quartz's clipboard handler reads `dataset.clipboard` (a JSON
69
+ // override) when present, otherwise falls back to `innerText`.
70
+ // Mirror that contract so syntax-highlighted blocks with line
71
+ // numbers don't leak the gutter into the copy buffer.
72
+ function codeTextFor(pre) {
73
+ var inner = pre.querySelector("code")
74
+ if (inner === null) return pre.innerText
75
+ if (inner.dataset.clipboard) {
76
+ try {
77
+ return JSON.parse(inner.dataset.clipboard)
78
+ } catch (_e) {
79
+ // Fall through to innerText if the JSON is malformed.
80
+ }
81
+ }
82
+ return inner.innerText
83
+ }
84
+
85
+ // brain: the action accessible name when the button is at rest.
86
+ // Mirrored into `aria-label` after every flash so SR users hear
87
+ // the right verb when the chip returns to its idle state.
88
+ var BASE_LABEL = "Copy code to clipboard"
89
+
90
+ // brain: the actual copy handler. Modern browsers gate
91
+ // `navigator.clipboard` behind secure contexts (https / localhost).
92
+ // Falls back to a no-op (visible "Copy failed" pulse) when the
93
+ // browser blocks it — better UX than a silent click.
94
+ function copyHandler(pre, button) {
95
+ return function (event) {
96
+ event.preventDefault()
97
+ var text = codeTextFor(pre)
98
+ if (typeof navigator.clipboard === "undefined") {
99
+ // Non-secure context — no clipboard API available.
100
+ flashLabel(button, "Unavailable")
101
+ return
102
+ }
103
+ navigator.clipboard.writeText(text).then(
104
+ function () {
105
+ flashLabel(button, "Copied")
106
+ },
107
+ function () {
108
+ flashLabel(button, "Failed")
109
+ },
110
+ )
111
+ }
112
+ }
113
+
114
+ // brain: swap the button label to a status string for 1.4s, then
115
+ // restore. Toggles `.is-copied` so `_code.scss` can paint the
116
+ // success state.
117
+ //
118
+ // brain (Lane C audit fix 2026-05-02): also mirror the status into
119
+ // `aria-label`. Per ARIA spec, `aria-label` OVERRIDES the
120
+ // element's text content for the accessible name — so updating
121
+ // only `textContent` left SR users hearing "Copy code to clipboard"
122
+ // on every press, regardless of whether the copy succeeded.
123
+ // Mapping: "Copied" → "Code copied to clipboard" (action-confirming
124
+ // affirmative), "Failed" / "Unavailable" → "<base label> — <status>"
125
+ // (annotated state) so the user hears why the action didn't take.
126
+ function flashLabel(button, status) {
127
+ var prior = button.dataset.brainOriginalLabel || "Copy"
128
+ button.textContent = status
129
+ var statusLabel
130
+ if (status === "Copied") {
131
+ statusLabel = "Code copied to clipboard"
132
+ } else {
133
+ statusLabel = BASE_LABEL + " — " + status.toLowerCase()
134
+ }
135
+ button.setAttribute("aria-label", statusLabel)
136
+ button.classList.add("is-copied")
137
+ window.setTimeout(function () {
138
+ button.textContent = prior
139
+ button.setAttribute("aria-label", BASE_LABEL)
140
+ button.classList.remove("is-copied")
141
+ }, 1400)
142
+ }
143
+
144
+ // brain: build the copy button as a DOM node. Class `.brain-code-copy`
145
+ // matches the SCSS selector in `_code.scss`. ARIA label spells out
146
+ // the action for screen readers (the visible "Copy" text is also
147
+ // the accessible name; `aria-label` refines it for non-visual
148
+ // users — and is updated on each flashLabel() call so it tracks
149
+ // the visible status).
150
+ function buildButton() {
151
+ var button = document.createElement("button")
152
+ button.type = "button"
153
+ button.className = "brain-code-copy"
154
+ button.textContent = "Copy"
155
+ button.setAttribute("aria-label", BASE_LABEL)
156
+ button.dataset.brainOriginalLabel = "Copy"
157
+ return button
158
+ }
159
+
160
+ // brain: walk every `<pre>` in the article body and inject the
161
+ // copy button if not already done. We scope to `article` so popover
162
+ // hints and chrome blocks (no code) aren't touched.
163
+ function injectAll() {
164
+ var pres = document.querySelectorAll("article pre")
165
+ pres.forEach(function (pre) {
166
+ // brain: idempotency — skip already-processed blocks.
167
+ if (pre.getAttribute(INJECTED_ATTR) === "true") return
168
+ pre.setAttribute(INJECTED_ATTR, "true")
169
+
170
+ // brain: ensure the language label has an attribute to read.
171
+ liftLanguageAttr(pre)
172
+
173
+ // brain: build + wire the button.
174
+ var button = buildButton()
175
+ button.addEventListener("click", copyHandler(pre, button))
176
+ pre.appendChild(button)
177
+ })
178
+ }
179
+
180
+ // brain: run on first DOM ready. `DOMContentLoaded` fires before
181
+ // first paint of below-fold content; above-fold pre blocks may
182
+ // briefly miss the button, but the hover-only opacity means it's
183
+ // invisible until cursor enters the pre — no flash.
184
+ if (document.readyState === "loading") {
185
+ document.addEventListener("DOMContentLoaded", injectAll)
186
+ } else {
187
+ injectAll()
188
+ }
189
+
190
+ // brain: re-run on Quartz SPA navigation. `enableSPA: true` in
191
+ // `quartz.config.ts` makes Quartz dispatch a `nav` event on
192
+ // `document` after each in-page route change. The injected button
193
+ // doesn't survive the morph (micromorph swaps the article DOM),
194
+ // so we re-walk on every nav.
195
+ document.addEventListener("nav", injectAll)
196
+ })()