secondbrain-py 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. brain/__init__.py +0 -0
  2. brain/__main__.py +18 -0
  3. brain/_capture_command.py +445 -0
  4. brain/_compose.py +52 -0
  5. brain/activity.py +206 -0
  6. brain/ask.py +631 -0
  7. brain/audio.py +591 -0
  8. brain/backfill/__init__.py +12 -0
  9. brain/backfill/search_extras.py +141 -0
  10. brain/backfill/source_rows.py +101 -0
  11. brain/bin/__init__.py +1 -0
  12. brain/bin/_launcher.py +107 -0
  13. brain/bin/down.py +8 -0
  14. brain/bin/launchd.py +268 -0
  15. brain/bin/monitor.py +570 -0
  16. brain/bin/rebuild.py +8 -0
  17. brain/bin/status.py +8 -0
  18. brain/bin/up.py +8 -0
  19. brain/brief.py +272 -0
  20. brain/capture.py +49 -0
  21. brain/chat.py +293 -0
  22. brain/cli.py +9760 -0
  23. brain/cli_claude.py +81 -0
  24. brain/cli_connect.py +285 -0
  25. brain/cli_demo.py +266 -0
  26. brain/config.py +1949 -0
  27. brain/connect.py +925 -0
  28. brain/db.py +540 -0
  29. brain/demo/__init__.py +452 -0
  30. brain/demo/corpus/manifest.json +403 -0
  31. brain/demo/embedder.py +74 -0
  32. brain/durations.py +84 -0
  33. brain/edit_session.py +156 -0
  34. brain/editor.py +67 -0
  35. brain/elicit/__init__.py +16 -0
  36. brain/elicit/detectors.py +250 -0
  37. brain/elicit/drafter.py +70 -0
  38. brain/elicit/queue.py +220 -0
  39. brain/elicit/schema.py +48 -0
  40. brain/elicit/session.py +445 -0
  41. brain/embedding_targets.py +54 -0
  42. brain/embeddings.py +424 -0
  43. brain/enrichment.py +808 -0
  44. brain/errors.py +357 -0
  45. brain/eval/__init__.py +129 -0
  46. brain/eval/answer_eval.py +281 -0
  47. brain/eval/baseline.py +265 -0
  48. brain/eval/concept_extraction.py +378 -0
  49. brain/eval/corpus.py +152 -0
  50. brain/eval/errors.py +19 -0
  51. brain/eval/graph_baseline.py +226 -0
  52. brain/eval/graph_retrieval.py +202 -0
  53. brain/eval/graph_runner.py +319 -0
  54. brain/eval/metrics.py +101 -0
  55. brain/eval/runner.py +223 -0
  56. brain/format.py +783 -0
  57. brain/gaps.py +390 -0
  58. brain/graph_rag/__init__.py +94 -0
  59. brain/graph_rag/_retrieval_common.py +113 -0
  60. brain/graph_rag/aggregates.py +303 -0
  61. brain/graph_rag/aliases/__init__.py +583 -0
  62. brain/graph_rag/backends/__init__.py +10 -0
  63. brain/graph_rag/backends/_age_helpers.py +473 -0
  64. brain/graph_rag/backends/age.py +782 -0
  65. brain/graph_rag/backends/base.py +272 -0
  66. brain/graph_rag/build.py +344 -0
  67. brain/graph_rag/communities.py +644 -0
  68. brain/graph_rag/communities_summary.py +437 -0
  69. brain/graph_rag/concepts.py +202 -0
  70. brain/graph_rag/cooccur.py +193 -0
  71. brain/graph_rag/cross_type.py +312 -0
  72. brain/graph_rag/extract.py +885 -0
  73. brain/graph_rag/fuse.py +371 -0
  74. brain/graph_rag/global_.py +412 -0
  75. brain/graph_rag/grouping.py +372 -0
  76. brain/graph_rag/person_resolver.py +167 -0
  77. brain/graph_rag/reconcile.py +792 -0
  78. brain/graph_rag/relational.py +353 -0
  79. brain/graph_rag/retrieve.py +526 -0
  80. brain/graph_rag/router.py +288 -0
  81. brain/graph_rag/schema.py +320 -0
  82. brain/graph_rag/sync.py +237 -0
  83. brain/graph_rag/tenancy.py +43 -0
  84. brain/graph_rag/themes.py +501 -0
  85. brain/graph_rag/weighting.py +202 -0
  86. brain/ingest/__init__.py +1926 -0
  87. brain/ingest/chunker.py +249 -0
  88. brain/ingest/docx.py +40 -0
  89. brain/ingest/gmail.py +621 -0
  90. brain/ingest/markdown.py +37 -0
  91. brain/ingest/pdf.py +61 -0
  92. brain/ingest/stdin.py +22 -0
  93. brain/ingest/sub_tokens.py +91 -0
  94. brain/ingest/text.py +16 -0
  95. brain/interactions.py +205 -0
  96. brain/maintenance.py +355 -0
  97. brain/mcp_server.py +3405 -0
  98. brain/migrations/001_init.sql +43 -0
  99. brain/migrations/002_qwen3_embedding.sql +17 -0
  100. brain/migrations/003_vault_model.sql +41 -0
  101. brain/migrations/004_relax_content_hash_uniqueness.sql +18 -0
  102. brain/migrations/005_derived_links.sql +67 -0
  103. brain/migrations/006_dedup_file_by_source_path.sql +25 -0
  104. brain/migrations/007_email_thread_and_draft.sql +15 -0
  105. brain/migrations/008_gmail_thread_unique.sql +11 -0
  106. brain/migrations/009_chunks_weighted_tsv.sql +28 -0
  107. brain/migrations/010_interactions.sql +30 -0
  108. brain/migrations/011_documents_summary.sql +23 -0
  109. brain/migrations/012_graphrag.sql +171 -0
  110. brain/migrations/013_graphrag_communities.sql +125 -0
  111. brain/migrations/014_graphrag_community_summary_hash.sql +33 -0
  112. brain/migrations/015_interactions_graph_targets.sql +89 -0
  113. brain/migrations/016_index_hygiene.sql +61 -0
  114. brain/migrations/017_elicit.sql +30 -0
  115. brain/migrations/018_review_gap_signal_kinds.sql +40 -0
  116. brain/migrations/019_search_queries.sql +35 -0
  117. brain/migrations/020_link_suggestions.sql +40 -0
  118. brain/migrations/021_timeline_doc_date.sql +34 -0
  119. brain/migrations/022_link_suggestions_undirected.sql +84 -0
  120. brain/migrations/023_search_queries_fts_count.sql +28 -0
  121. brain/quartz_overrides/__init__.py +8 -0
  122. brain/quartz_overrides/quartz/bootstrap-cli.mjs +65 -0
  123. brain/quartz_overrides/quartz/build.ts +568 -0
  124. brain/quartz_overrides/quartz/cli/args.js +152 -0
  125. brain/quartz_overrides/quartz/cli/build_partial_handler.js +544 -0
  126. brain/quartz_overrides/quartz/cli/handlers.js +636 -0
  127. brain/quartz_overrides/quartz/components/CommandPalette.tsx +172 -0
  128. brain/quartz_overrides/quartz/components/Explorer.tsx +198 -0
  129. brain/quartz_overrides/quartz/components/Footer.tsx +27 -0
  130. brain/quartz_overrides/quartz/components/Graph.tsx +468 -0
  131. brain/quartz_overrides/quartz/components/PageTitle.tsx +72 -0
  132. brain/quartz_overrides/quartz/components/RelatedDocs.tsx +38 -0
  133. brain/quartz_overrides/quartz/components/Search.tsx +161 -0
  134. brain/quartz_overrides/quartz/components/SummaryLede.tsx +72 -0
  135. brain/quartz_overrides/quartz/components/index.ts +92 -0
  136. brain/quartz_overrides/quartz/components/pages/TagContent.tsx +272 -0
  137. brain/quartz_overrides/quartz/components/scripts/commandPalette.inline.ts +665 -0
  138. brain/quartz_overrides/quartz/components/scripts/explorer.inline.ts +768 -0
  139. brain/quartz_overrides/quartz/components/scripts/graph.inline.ts +2302 -0
  140. brain/quartz_overrides/quartz/components/scripts/relatedDocs.inline.ts +163 -0
  141. brain/quartz_overrides/quartz/components/scripts/search.inline.ts +1011 -0
  142. brain/quartz_overrides/quartz/plugins/emitters/contentIndex.ts +546 -0
  143. brain/quartz_overrides/quartz/plugins/transformers/codeCopy.ts +94 -0
  144. brain/quartz_overrides/quartz/plugins/transformers/derivedFenceMark.ts +302 -0
  145. brain/quartz_overrides/quartz/plugins/transformers/emailThread.ts +148 -0
  146. brain/quartz_overrides/quartz/plugins/transformers/emptyDoorFilter.ts +213 -0
  147. brain/quartz_overrides/quartz/plugins/transformers/index.ts +114 -0
  148. brain/quartz_overrides/quartz/plugins/transformers/linkKindMark.ts +205 -0
  149. brain/quartz_overrides/quartz/plugins/transformers/linkSourceTag.ts +104 -0
  150. brain/quartz_overrides/quartz/plugins/transformers/relativeDate.ts +100 -0
  151. brain/quartz_overrides/quartz/plugins/transformers/reloadSignal.ts +131 -0
  152. brain/quartz_overrides/quartz/processors/parse.ts +371 -0
  153. brain/quartz_overrides/quartz/processors/parser_cache.ts +78 -0
  154. brain/quartz_overrides/quartz/static/brain-logo-dark.png +0 -0
  155. brain/quartz_overrides/quartz/static/brain-logo-light.png +0 -0
  156. brain/quartz_overrides/quartz/static/codeCopy.js +196 -0
  157. brain/quartz_overrides/quartz/static/emailThread.js +334 -0
  158. brain/quartz_overrides/quartz/static/favicon.ico +0 -0
  159. brain/quartz_overrides/quartz/static/icon.png +0 -0
  160. brain/quartz_overrides/quartz/static/linkSourceTag.js +104 -0
  161. brain/quartz_overrides/quartz/static/relativeDate.js +142 -0
  162. brain/quartz_overrides/quartz/static/reload.js +168 -0
  163. brain/quartz_overrides/quartz/styles/brain/_article.scss +252 -0
  164. brain/quartz_overrides/quartz/styles/brain/_atmosphere.scss +113 -0
  165. brain/quartz_overrides/quartz/styles/brain/_callouts.scss +180 -0
  166. brain/quartz_overrides/quartz/styles/brain/_cmdk.scss +7 -0
  167. brain/quartz_overrides/quartz/styles/brain/_code.scss +208 -0
  168. brain/quartz_overrides/quartz/styles/brain/_command_palette.scss +369 -0
  169. brain/quartz_overrides/quartz/styles/brain/_email_thread.scss +228 -0
  170. brain/quartz_overrides/quartz/styles/brain/_explorer.scss +142 -0
  171. brain/quartz_overrides/quartz/styles/brain/_home.scss +182 -0
  172. brain/quartz_overrides/quartz/styles/brain/_links.scss +322 -0
  173. brain/quartz_overrides/quartz/styles/brain/_marginalia.scss +117 -0
  174. brain/quartz_overrides/quartz/styles/brain/_motion.scss +175 -0
  175. brain/quartz_overrides/quartz/styles/brain/_people_hub.scss +100 -0
  176. brain/quartz_overrides/quartz/styles/brain/_related_docs.scss +137 -0
  177. brain/quartz_overrides/quartz/styles/brain/_search.scss +252 -0
  178. brain/quartz_overrides/quartz/styles/brain/_sidebar.scss +468 -0
  179. brain/quartz_overrides/quartz/styles/brain/_summary_lede.scss +56 -0
  180. brain/quartz_overrides/quartz/styles/brain/_surface.scss +43 -0
  181. brain/quartz_overrides/quartz/styles/brain/_tag_content.scss +118 -0
  182. brain/quartz_overrides/quartz/styles/brain/_tokens.scss +197 -0
  183. brain/quartz_overrides/quartz/styles/brain/_typography.scss +92 -0
  184. brain/quartz_overrides/quartz/styles/custom.scss +89 -0
  185. brain/quartz_overrides/quartz/styles/graph.scss +505 -0
  186. brain/quartz_overrides/quartz/util/ctx.ts +92 -0
  187. brain/quartz_overrides/quartz/util/fastpath_manifest.ts +608 -0
  188. brain/quartz_overrides/quartz/util/path.ts +358 -0
  189. brain/quartz_overrides/quartz/util/sourceIcons.ts +55 -0
  190. brain/quartz_overrides/quartz.config.ts +270 -0
  191. brain/quartz_overrides/quartz.layout.ts +314 -0
  192. brain/queries.py +1188 -0
  193. brain/rank_fusion.py +8 -0
  194. brain/resurface.py +210 -0
  195. brain/review/__init__.py +26 -0
  196. brain/review/emit.py +27 -0
  197. brain/review/queries.py +436 -0
  198. brain/review/render.py +196 -0
  199. brain/review/scans.py +355 -0
  200. brain/review/weekly.py +413 -0
  201. brain/search.py +704 -0
  202. brain/set_similarity.py +15 -0
  203. brain/setup.py +1205 -0
  204. brain/tags.py +56 -0
  205. brain/templates/Caddyfile.j2 +9 -0
  206. brain/templates/__init__.py +1 -0
  207. brain/templates/bin/__init__.py +1 -0
  208. brain/templates/bin/_brain-brief-fg.sh +25 -0
  209. brain/templates/bin/_brain-build-fg.sh +53 -0
  210. brain/templates/bin/_brain-watcher-fg.sh +65 -0
  211. brain/templates/bin/brain-down.sh +89 -0
  212. brain/templates/bin/brain-status.sh +83 -0
  213. brain/templates/bin/brain-up.sh +221 -0
  214. brain/templates/docker/age/Dockerfile +79 -0
  215. brain/templates/docker-compose.stock.yml.j2 +26 -0
  216. brain/templates/docker-compose.yml.j2 +34 -0
  217. brain/templates/env.example +190 -0
  218. brain/templates/launchd/__init__.py +1 -0
  219. brain/templates/launchd/com.brain.brief.plist.j2 +45 -0
  220. brain/templates/launchd/com.brain.build.plist.j2 +46 -0
  221. brain/templates/launchd/com.brain.watcher.plist.j2 +46 -0
  222. brain/templates/skill/SKILL.md +63 -0
  223. brain/templates/skill/__init__.py +1 -0
  224. brain/timeline.py +834 -0
  225. brain/todo.py +124 -0
  226. brain/uninstall.py +185 -0
  227. brain/vault/__init__.py +115 -0
  228. brain/vault/_atomic.py +25 -0
  229. brain/vault/daily_index.py +228 -0
  230. brain/vault/derived_links/__init__.py +50 -0
  231. brain/vault/derived_links/directory.py +683 -0
  232. brain/vault/derived_links/fence.py +408 -0
  233. brain/vault/derived_links/gws.py +64 -0
  234. brain/vault/derived_links/participants.py +143 -0
  235. brain/vault/derived_links/pass_runner.py +362 -0
  236. brain/vault/derived_links/rules.py +137 -0
  237. brain/vault/export.py +683 -0
  238. brain/vault/frontmatter.py +165 -0
  239. brain/vault/graph.py +620 -0
  240. brain/vault/graph_format.py +388 -0
  241. brain/vault/link_rewrite.py +235 -0
  242. brain/vault/links.py +260 -0
  243. brain/vault/note_builder.py +211 -0
  244. brain/vault/paths.py +55 -0
  245. brain/vault/quartz_overlay.py +236 -0
  246. brain/vault/rename.py +591 -0
  247. brain/vault/resolver.py +304 -0
  248. brain/vault/slug.py +127 -0
  249. brain/vault/sync.py +1513 -0
  250. brain/vault/sync_summaries.py +264 -0
  251. brain/vault/templates.py +145 -0
  252. brain/vault/watch.py +1052 -0
  253. brain/wiki/__init__.py +6 -0
  254. brain/wiki/_github_slugger.py +76 -0
  255. brain/wiki/_person_name.py +314 -0
  256. brain/wiki/build_homepage.py +541 -0
  257. brain/wiki/build_partial.py +273 -0
  258. brain/wiki/build_people.py +934 -0
  259. brain/wiki/build_related.py +758 -0
  260. brain/wiki/build_swap.py +585 -0
  261. brain/wiki/build_watcher.py +975 -0
  262. brain/wiki/edit_classifier.py +215 -0
  263. brain/wiki/errors.py +10 -0
  264. brain/wiki/fastpath_manifest.py +475 -0
  265. brain/wiki/fastpath_state.py +174 -0
  266. brain/wiki/install.py +296 -0
  267. brain/wiki/slug.py +111 -0
  268. secondbrain_py-0.2.1.dist-info/METADATA +195 -0
  269. secondbrain_py-0.2.1.dist-info/RECORD +273 -0
  270. secondbrain_py-0.2.1.dist-info/WHEEL +5 -0
  271. secondbrain_py-0.2.1.dist-info/entry_points.txt +11 -0
  272. secondbrain_py-0.2.1.dist-info/licenses/LICENSE +21 -0
  273. secondbrain_py-0.2.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,644 @@
1
+ """Global community detection over the tenant entity graph (wave G3-b, spec §17c).
2
+
3
+ The callable CORE of ``brain graphrag communities build|refresh`` (the CLI/MCP
4
+ surfaces land in G3-f). No summaries, no embeddings, no retrieval — just the
5
+ networkx-Louvain partition + Jaccard stable-identity + dirty/delta gating and
6
+ the relational persistence of :class:`~brain.graph_rag.schema.CommunityRecord` /
7
+ :class:`~brain.graph_rag.schema.CommunityMember` rows.
8
+
9
+ **RELATIONAL-only (§17c Q2).** The input graph is built from a single
10
+ tenant-scoped read of ``graph_relationships`` (the lift-weighted edge mirror,
11
+ migration 012) — NOT an AGE traversal — and the output communities live entirely
12
+ in ``graph_communities`` / ``graph_community_members`` (migration 013). AGE keeps
13
+ only ``Entity``/``Document`` + ``MENTIONED_IN``/``CO_OCCURS``; there is no
14
+ ``Community`` vertex.
15
+
16
+ **Dirty gate (§17c Q3).** :func:`build_communities` computes the tenant's
17
+ ``source_graph_hash`` — a deterministic hash over the ordered edge set — and
18
+ SKIPS (no-op) when the stored communities' ``(build_version, source_graph_hash)``
19
+ already matches and ``force`` is False. ``communities refresh`` (force=True)
20
+ bypasses the gate. Detection / Jaccard matching run ONLY after the gate fires;
21
+ there is no per-query Louvain (batched at build/refresh per LazyGraphRAG §4 D2).
22
+ Because the fingerprint is carried ON the community rows, a tenant with zero
23
+ materialized communities (empty graph, or every partition below ``min_size``) is
24
+ always re-evaluated — which is cheap (Louvain over an empty/tiny graph).
25
+
26
+ **Stable identity (§17c Q3/Q7).** Each newly-detected community's member set is
27
+ greedy-best-Jaccard matched (threshold ``BRAIN_GRAPH_COMMUNITY_JACCARD``) against
28
+ the EXISTING stored communities for the tenant; a match at or above the threshold
29
+ (and with non-zero overlap) reuses that ``community_key`` — preserving the
30
+ summary row — while an unmatched community mints a fresh UUID. ``members_hash``
31
+ (a deterministic hash over the sorted member entity ids) is the per-community
32
+ identity. The Jaccard helper is the shared :func:`brain.set_similarity.jaccard`
33
+ (§17c Q7).
34
+
35
+ **Summary delta-gate (§17c Q3/Q10).** A reused row is UPDATEd in place, which
36
+ NEVER touches ``summary`` / ``summary_model`` / ``summary_at`` /
37
+ ``summary_embedding``: a still-valid summary is preserved across rebuilds, and a
38
+ membership change is recorded via the updated ``members_hash`` so G3-c can refresh
39
+ the (now stale) summary later. Minted rows start with NULL summary fields.
40
+
41
+ **Determinism.** Louvain is randomized, so the configured ``seed`` is threaded
42
+ through and the partition + member ordering are deterministically sorted
43
+ (communities by ``members_hash``; members by descending weighted degree then
44
+ entity id). Combined with the dirty gate (a second build on an unchanged graph
45
+ SKIPS) and Jaccard key reuse, repeated builds on the same graph converge to
46
+ byte-stable rows (timestamps aside).
47
+ """
48
+ from __future__ import annotations
49
+
50
+ import hashlib
51
+ import uuid
52
+ from collections.abc import Iterable, Sequence
53
+ from dataclasses import dataclass
54
+ from typing import Any
55
+
56
+ import networkx as nx
57
+ import psycopg
58
+ from networkx.algorithms.community import louvain_communities
59
+
60
+ from ..config import Config
61
+ from ..errors import GraphTenantError
62
+ from ..set_similarity import jaccard
63
+ from .schema import CommunityMember, CommunityRecord
64
+
65
+ __all__ = [
66
+ "BUILD_VERSION",
67
+ "CommunityBuildResult",
68
+ "DetectedCommunity",
69
+ "build_communities",
70
+ "compute_members_hash",
71
+ "compute_source_graph_hash",
72
+ "detect_communities",
73
+ "list_communities",
74
+ "match_communities",
75
+ ]
76
+
77
+ # Detection algorithm version. MUST match the ``graph_communities.build_version``
78
+ # DB default (migration 013) so the dirty gate's stored-fingerprint comparison
79
+ # is meaningful. Bump when the partitioning semantics change (a new Louvain
80
+ # variant, a different weighting input) so a rebuild is forced corpus-wide.
81
+ BUILD_VERSION = "networkx-louvain-v1"
82
+
83
+
84
+ @dataclass(frozen=True)
85
+ class DetectedCommunity:
86
+ """One community produced by :func:`detect_communities` (pre-persistence).
87
+
88
+ ``members`` are the ranked :class:`~brain.graph_rag.schema.CommunityMember`s
89
+ WITHOUT an assigned ``community_key`` (the key is matched/minted at persist).
90
+ ``members_hash`` is the per-community identity over the sorted member ids;
91
+ the aggregate stats describe the community subgraph.
92
+ """
93
+
94
+ members_hash: str
95
+ members: tuple[CommunityMember, ...]
96
+ member_count: int
97
+ edge_count: int
98
+ total_weight: float
99
+
100
+ @property
101
+ def member_ids(self) -> frozenset[str]:
102
+ """The community's entity-id set (for Jaccard stable-identity matching)."""
103
+ return frozenset(member.entity_id for member in self.members)
104
+
105
+
106
+ @dataclass(frozen=True)
107
+ class CommunityBuildResult:
108
+ """Tally of a :func:`build_communities` run.
109
+
110
+ ``communities_total`` is the number of communities materialized for the
111
+ tenant after the run (the stored count when ``skipped``). ``created`` /
112
+ ``reused`` / ``deleted`` partition the change: minted keys, Jaccard-reused
113
+ keys, and removed keys (no longer present). ``dirty`` is True when the
114
+ tenant's ``source_graph_hash`` differs from the stored fingerprint (the graph
115
+ genuinely changed since the last build) — independent of ``force``, so a
116
+ forced rebuild of an unchanged graph reports ``dirty=False`` with
117
+ ``skipped=False``. ``skipped`` is True only when the dirty gate fired (graph
118
+ unchanged AND not forced) and no work ran.
119
+ """
120
+
121
+ tenant_id: str
122
+ source_graph_hash: str = ""
123
+ communities_total: int = 0
124
+ created: int = 0
125
+ reused: int = 0
126
+ deleted: int = 0
127
+ dirty: bool = False
128
+ skipped: bool = False
129
+
130
+
131
+ # --------------------------------------------------------------------------- #
132
+ # Pure helpers (no DB) — hashing, detection, stable-identity matching.
133
+ # --------------------------------------------------------------------------- #
134
+ def compute_members_hash(member_ids: Iterable[str]) -> str:
135
+ """Deterministic per-community identity hash over the member entity ids.
136
+
137
+ Sorts the ids (so call-order never matters) and SHA-256s the newline-joined
138
+ string. Entity ids are UUID text (no newline), so the join is unambiguous.
139
+ """
140
+ joined = "\n".join(sorted(str(mid) for mid in member_ids))
141
+ return hashlib.sha256(joined.encode("utf-8")).hexdigest()
142
+
143
+
144
+ def compute_source_graph_hash(edges: Iterable[tuple[str, str, float]]) -> str:
145
+ """Deterministic dirty fingerprint over the tenant's edge set (§17c Q3).
146
+
147
+ Sorts the ``(src, dst, weight)`` triples by ``(src, dst)`` (the endpoints are
148
+ already canonical ``src < dst`` per migration 012) and SHA-256s the
149
+ tab/newline-joined rendering. ``weight`` is rendered with ``repr`` — Python's
150
+ shortest round-trippable float string, which is injective over floats — so a
151
+ genuine weight change always moves the hash and an equal weight never
152
+ spuriously does (same rationale as ``weighting.suppress_ver``). An empty edge
153
+ set hashes the empty string (a fixed constant), so a graph with no
154
+ relationships has a stable, well-defined fingerprint.
155
+ """
156
+ parts = [
157
+ f"{src}\t{dst}\t{weight!r}"
158
+ for src, dst, weight in sorted(edges, key=lambda edge: (edge[0], edge[1]))
159
+ ]
160
+ return hashlib.sha256("\n".join(parts).encode("utf-8")).hexdigest()
161
+
162
+
163
+ def detect_communities(
164
+ edges: Sequence[tuple[str, str, float]],
165
+ *,
166
+ resolution: float,
167
+ seed: int,
168
+ min_size: int,
169
+ max_communities: int | None = None,
170
+ ) -> list[DetectedCommunity]:
171
+ """Partition the weighted edge set into communities (pure; no DB).
172
+
173
+ Builds an undirected weighted :class:`networkx.Graph` from ``edges``, runs
174
+ ``louvain_communities`` with the configured ``resolution`` + ``seed``, DROPS
175
+ partitions smaller than ``min_size`` (sub-threshold groups are not
176
+ materialized — spec §6c / the ``BRAIN_GRAPH_COMMUNITY_MIN_SIZE`` knob), and
177
+ computes each surviving community's identity hash + ranked members + subgraph
178
+ stats. When ``max_communities`` is set and exceeded, the LARGEST communities
179
+ are kept (by member count, then total weight, then ``members_hash`` for a
180
+ deterministic tie-break) — the ops cap bounds the downstream summary +
181
+ embedding cost (§17c Q8). The returned list is sorted by ``members_hash`` for
182
+ byte-stable output independent of Louvain's internal partition order.
183
+ """
184
+ graph: nx.Graph = nx.Graph()
185
+ for src, dst, weight in edges:
186
+ graph.add_edge(str(src), str(dst), weight=float(weight))
187
+ if graph.number_of_nodes() == 0:
188
+ return []
189
+
190
+ partitions = louvain_communities(
191
+ graph, weight="weight", resolution=resolution, seed=seed
192
+ )
193
+
194
+ detected: list[DetectedCommunity] = []
195
+ for nodes in partitions:
196
+ if len(nodes) < min_size:
197
+ continue
198
+ detected.append(_build_detected(graph, nodes))
199
+
200
+ # Ops safety valve (§17c Q8): keep the largest communities, deterministically.
201
+ if max_communities is not None and len(detected) > max_communities:
202
+ detected = sorted(
203
+ detected,
204
+ key=lambda dc: (-dc.member_count, -dc.total_weight, dc.members_hash),
205
+ )[:max_communities]
206
+
207
+ # Byte-stable output order, independent of Louvain's partition ordering.
208
+ detected.sort(key=lambda dc: dc.members_hash)
209
+ return detected
210
+
211
+
212
+ def _build_detected(graph: nx.Graph, nodes: Iterable[str]) -> DetectedCommunity:
213
+ """Assemble one :class:`DetectedCommunity` from a Louvain partition.
214
+
215
+ Members are ranked by descending weighted degree within the community
216
+ subgraph, ties broken by entity id (ascending) so the ordering is
217
+ deterministic. ``edge_count`` / ``total_weight`` describe the induced
218
+ subgraph.
219
+ """
220
+ member_ids = sorted(str(node) for node in nodes)
221
+ sub = graph.subgraph(member_ids)
222
+ weighted_degree = {str(node): float(deg) for node, deg in sub.degree(weight="weight")}
223
+ ranked_ids = sorted(member_ids, key=lambda mid: (-weighted_degree.get(mid, 0.0), mid))
224
+ members = tuple(
225
+ CommunityMember(
226
+ entity_id=mid,
227
+ member_rank=rank,
228
+ member_weight=weighted_degree.get(mid, 0.0),
229
+ )
230
+ for rank, mid in enumerate(ranked_ids)
231
+ )
232
+ total_weight = float(sum(float(weight) for _, _, weight in sub.edges(data="weight")))
233
+ return DetectedCommunity(
234
+ members_hash=compute_members_hash(member_ids),
235
+ members=members,
236
+ member_count=len(member_ids),
237
+ edge_count=sub.number_of_edges(),
238
+ total_weight=total_weight,
239
+ )
240
+
241
+
242
+ def match_communities(
243
+ detected: Sequence[DetectedCommunity],
244
+ existing: Sequence[tuple[str, frozenset[str]]],
245
+ *,
246
+ threshold: float,
247
+ ) -> tuple[list[str | None], list[str]]:
248
+ """Greedy best-Jaccard stable-identity match (§17c Q3/Q7; pure, no DB).
249
+
250
+ For each detected community (processed in the caller's deterministic order),
251
+ selects the UNCLAIMED existing community with the highest Jaccard overlap of
252
+ member sets (ties broken by ``community_key`` ascending). A best match at or
253
+ above ``threshold`` AND with non-zero overlap REUSES that key (and claims it,
254
+ so two new communities never collapse onto one old key); otherwise the
255
+ community is new (``None`` → the caller mints a UUID). Requiring strictly
256
+ positive overlap means a zero-overlap pairing is never reused even at
257
+ ``threshold == 0`` — a disjoint set is a different community.
258
+
259
+ Returns ``(assigned_keys, deleted_keys)``: ``assigned_keys`` is parallel to
260
+ ``detected`` (reused key string or ``None`` to mint); ``deleted_keys`` are
261
+ the existing keys never claimed (no longer present → delete + CASCADE members).
262
+ """
263
+ existing_list = list(existing)
264
+ claimed: set[int] = set()
265
+ assigned: list[str | None] = []
266
+ for community in detected:
267
+ target = community.member_ids
268
+ best: tuple[float, str, int] | None = None
269
+ for index, (key, members) in enumerate(existing_list):
270
+ if index in claimed:
271
+ continue
272
+ score = jaccard(target, members)
273
+ if best is None or score > best[0] or (score == best[0] and key < best[1]):
274
+ best = (score, key, index)
275
+ if best is not None and best[0] >= threshold and best[0] > 0.0:
276
+ assigned.append(best[1])
277
+ claimed.add(best[2])
278
+ else:
279
+ assigned.append(None)
280
+ deleted = [
281
+ key for index, (key, _members) in enumerate(existing_list) if index not in claimed
282
+ ]
283
+ return assigned, deleted
284
+
285
+
286
+ # --------------------------------------------------------------------------- #
287
+ # Persistence — tenant-scoped, atomic (mirrors aggregates.py / relational.py).
288
+ # --------------------------------------------------------------------------- #
289
+ def build_communities(
290
+ conn: psycopg.Connection[Any],
291
+ cfg: Config,
292
+ *,
293
+ tenant: str,
294
+ force: bool = False,
295
+ ) -> CommunityBuildResult:
296
+ """Detect + persist the tenant's communities (the G3-b core).
297
+
298
+ Reads the tenant's ``graph_relationships`` edge set, computes the
299
+ ``source_graph_hash`` dirty fingerprint, and — unless ``force`` — SKIPS when
300
+ the stored ``(build_version, source_graph_hash)`` already matches (§17c Q3).
301
+ Otherwise runs Louvain (seeded + sorted for determinism), Jaccard-matches the
302
+ partition against the existing communities to preserve ``community_key``s
303
+ (and their summaries), and atomically replaces the tenant's
304
+ ``graph_communities`` + ``graph_community_members`` rows inside a single
305
+ ``conn.transaction()`` (the connection may be autocommit; the explicit
306
+ transaction brackets the read-modify-write as a unit, mirroring
307
+ :func:`brain.graph_rag.aggregates.refresh_aggregates`).
308
+
309
+ Community knobs are threaded from ``cfg``:
310
+ ``graph_community_resolution`` / ``_seed`` / ``_min_size`` / ``_jaccard`` /
311
+ ``_max``. ``tenant`` must be non-empty (the caller resolves it via
312
+ :func:`brain.graph_rag.tenancy.resolve_tenant`); an empty tenant is a caller
313
+ bug and raises :class:`brain.errors.GraphTenantError` before any DB work.
314
+ """
315
+ if not tenant:
316
+ raise GraphTenantError(
317
+ "build_communities requires a non-empty tenant_id "
318
+ "(resolve via brain.graph_rag.tenancy.resolve_tenant first)"
319
+ )
320
+
321
+ with conn.transaction():
322
+ edges = _read_edges(conn, tenant)
323
+ source_graph_hash = compute_source_graph_hash(edges)
324
+ matches = _fingerprint_matches(conn, tenant, source_graph_hash)
325
+ dirty = not matches
326
+
327
+ if matches and not force:
328
+ # Dirty gate fired: graph unchanged and not forced → no-op.
329
+ return CommunityBuildResult(
330
+ tenant_id=tenant,
331
+ source_graph_hash=source_graph_hash,
332
+ communities_total=_stored_count(conn, tenant),
333
+ dirty=False,
334
+ skipped=True,
335
+ )
336
+
337
+ detected = detect_communities(
338
+ edges,
339
+ resolution=cfg.graph_community_resolution,
340
+ seed=cfg.graph_community_seed,
341
+ min_size=cfg.graph_community_min_size,
342
+ max_communities=cfg.graph_community_max,
343
+ )
344
+ existing = _read_existing_communities(conn, tenant)
345
+ assigned, deleted_keys = match_communities(
346
+ detected,
347
+ [(key, members) for key, members, _hash in existing],
348
+ threshold=cfg.graph_community_jaccard,
349
+ )
350
+ created, reused = _persist(
351
+ conn,
352
+ tenant,
353
+ source_graph_hash=source_graph_hash,
354
+ detected=detected,
355
+ assigned=assigned,
356
+ deleted_keys=deleted_keys,
357
+ )
358
+ return CommunityBuildResult(
359
+ tenant_id=tenant,
360
+ source_graph_hash=source_graph_hash,
361
+ communities_total=len(detected),
362
+ created=created,
363
+ reused=reused,
364
+ deleted=len(deleted_keys),
365
+ dirty=dirty,
366
+ skipped=False,
367
+ )
368
+
369
+
370
+ def list_communities(
371
+ conn: psycopg.Connection[Any],
372
+ tenant: str,
373
+ *,
374
+ limit: int | None = None,
375
+ ) -> list[CommunityRecord]:
376
+ """Read the tenant's materialized communities (the admin-listing read; G3-f).
377
+
378
+ Returns the stored :class:`~brain.graph_rag.schema.CommunityRecord` rows for
379
+ ``tenant``, ordered by ``member_count`` DESC then ``community_key`` so the
380
+ largest communities surface first (deterministic tie-break), capped by
381
+ ``limit`` when given. Read-only — the raw ``summary_embedding`` vector is not
382
+ selected (a storage handle, not a wire value). ``tenant`` must be non-empty
383
+ (the caller resolves it via :func:`brain.graph_rag.tenancy.resolve_tenant`).
384
+ """
385
+ if not tenant:
386
+ raise GraphTenantError(
387
+ "list_communities requires a non-empty tenant_id "
388
+ "(resolve via brain.graph_rag.tenancy.resolve_tenant first)"
389
+ )
390
+ base = (
391
+ "SELECT community_key::text, source_graph_hash, members_hash, level, "
392
+ "build_version, member_count, edge_count, total_weight, summary, "
393
+ "summary_model, summary_at "
394
+ "FROM graph_communities WHERE tenant_id = %s "
395
+ "ORDER BY member_count DESC, community_key"
396
+ )
397
+ if limit is not None:
398
+ rows = conn.execute(base + " LIMIT %s", (tenant, limit)).fetchall()
399
+ else:
400
+ rows = conn.execute(base, (tenant,)).fetchall()
401
+ return [
402
+ CommunityRecord(
403
+ community_key=str(row[0]),
404
+ source_graph_hash=str(row[1]),
405
+ members_hash=str(row[2]),
406
+ tenant_id=tenant,
407
+ level=int(row[3]),
408
+ build_version=str(row[4]),
409
+ member_count=int(row[5]),
410
+ edge_count=int(row[6]),
411
+ total_weight=float(row[7]),
412
+ summary=row[8],
413
+ summary_model=row[9],
414
+ summary_at=row[10],
415
+ )
416
+ for row in rows
417
+ ]
418
+
419
+
420
+ def _read_edges(
421
+ conn: psycopg.Connection[Any], tenant: str
422
+ ) -> list[tuple[str, str, float]]:
423
+ """Read the tenant's ``graph_relationships`` edge set, ordered for hashing."""
424
+ rows = conn.execute(
425
+ "SELECT src_id::text, dst_id::text, weight FROM graph_relationships "
426
+ "WHERE tenant_id = %s ORDER BY src_id, dst_id",
427
+ (tenant,),
428
+ ).fetchall()
429
+ return [(str(src), str(dst), float(weight)) for src, dst, weight in rows]
430
+
431
+
432
+ def _fingerprint_matches(
433
+ conn: psycopg.Connection[Any], tenant: str, source_graph_hash: str
434
+ ) -> bool:
435
+ """True iff the stored communities all carry the current fingerprint.
436
+
437
+ The dirty gate (§17c Q3). Reads the DISTINCT ``(build_version,
438
+ source_graph_hash)`` for the tenant's communities; a match requires exactly
439
+ one distinct pair equal to ``(BUILD_VERSION, source_graph_hash)``. Zero rows
440
+ (no prior build / a zero-community tenant) is NOT a match — the build always
441
+ re-evaluates, which is cheap.
442
+ """
443
+ rows = conn.execute(
444
+ "SELECT DISTINCT build_version, source_graph_hash FROM graph_communities "
445
+ "WHERE tenant_id = %s",
446
+ (tenant,),
447
+ ).fetchall()
448
+ return rows == [(BUILD_VERSION, source_graph_hash)]
449
+
450
+
451
+ def _stored_count(conn: psycopg.Connection[Any], tenant: str) -> int:
452
+ """Count the tenant's materialized communities (for the skipped report)."""
453
+ row = conn.execute(
454
+ "SELECT COUNT(*) FROM graph_communities WHERE tenant_id = %s",
455
+ (tenant,),
456
+ ).fetchone()
457
+ return int(row[0]) if row is not None else 0
458
+
459
+
460
+ def _read_existing_communities(
461
+ conn: psycopg.Connection[Any], tenant: str
462
+ ) -> list[tuple[str, frozenset[str], str]]:
463
+ """Read existing communities as ``(community_key, member_ids, members_hash)``.
464
+
465
+ Used both for Jaccard stable-identity matching (member sets) and to decide
466
+ which keys are reused vs deleted. Two reads (communities + members) joined in
467
+ Python keep the SQL trivial and tenant-scoped.
468
+ """
469
+ community_rows = conn.execute(
470
+ "SELECT community_key::text, members_hash FROM graph_communities "
471
+ "WHERE tenant_id = %s",
472
+ (tenant,),
473
+ ).fetchall()
474
+ member_rows = conn.execute(
475
+ "SELECT community_key::text, entity_id::text FROM graph_community_members "
476
+ "WHERE tenant_id = %s",
477
+ (tenant,),
478
+ ).fetchall()
479
+ members_by_key: dict[str, set[str]] = {}
480
+ for key, entity_id in member_rows:
481
+ members_by_key.setdefault(str(key), set()).add(str(entity_id))
482
+ return [
483
+ (str(key), frozenset(members_by_key.get(str(key), set())), str(members_hash))
484
+ for key, members_hash in community_rows
485
+ ]
486
+
487
+
488
+ def _persist(
489
+ conn: psycopg.Connection[Any],
490
+ tenant: str,
491
+ *,
492
+ source_graph_hash: str,
493
+ detected: Sequence[DetectedCommunity],
494
+ assigned: Sequence[str | None],
495
+ deleted_keys: Sequence[str],
496
+ ) -> tuple[int, int]:
497
+ """Atomically replace the tenant's communities + members. Returns (created, reused).
498
+
499
+ Order (within the caller's transaction):
500
+
501
+ 1. DELETE communities no longer present (CASCADE clears their members).
502
+ 2. For every REUSED community, delete its members and set a temporary, unique
503
+ ``members_hash`` sentinel. This two-pass dance dodges a transient
504
+ ``UNIQUE (tenant_id, level, members_hash)`` violation: a final hash being
505
+ assigned to one row could otherwise momentarily collide with another
506
+ reused row's not-yet-updated old hash.
507
+ 3. For every reused community, write the FINAL ``members_hash`` + stats +
508
+ fingerprint. The summary columns are intentionally absent from the SET
509
+ clause, so a reused row's summary/embedding is PRESERVED (delta-gate;
510
+ §17c Q3/Q10) while a membership change is recorded via ``members_hash``.
511
+ 4. INSERT minted communities (NULL summary fields).
512
+ 5. INSERT members for every kept community (reused + minted).
513
+ """
514
+ if deleted_keys:
515
+ conn.execute(
516
+ "DELETE FROM graph_communities "
517
+ "WHERE tenant_id = %s AND community_key::text = ANY(%s)",
518
+ (tenant, list(deleted_keys)),
519
+ )
520
+
521
+ reused_keys = [key for key in assigned if key is not None]
522
+
523
+ # Pass 2a: clear members + park reused rows on a unique sentinel hash.
524
+ for reused_key in reused_keys:
525
+ conn.execute(
526
+ "DELETE FROM graph_community_members "
527
+ "WHERE tenant_id = %s AND community_key = %s",
528
+ (tenant, reused_key),
529
+ )
530
+ conn.execute(
531
+ "UPDATE graph_communities "
532
+ "SET members_hash = %s WHERE tenant_id = %s AND community_key = %s",
533
+ (f"pending:{reused_key}", tenant, reused_key),
534
+ )
535
+
536
+ created = 0
537
+ reused = 0
538
+ # ``members_to_insert`` collects (community_key, member) once the key is known.
539
+ members_to_insert: list[tuple[str, CommunityMember]] = []
540
+ for community, key in zip(detected, assigned, strict=True):
541
+ if key is None:
542
+ community_key = str(uuid.uuid4())
543
+ _insert_community(
544
+ conn,
545
+ tenant,
546
+ community_key=community_key,
547
+ source_graph_hash=source_graph_hash,
548
+ community=community,
549
+ )
550
+ created += 1
551
+ else:
552
+ community_key = key
553
+ _update_reused_community(
554
+ conn,
555
+ tenant,
556
+ community_key=community_key,
557
+ source_graph_hash=source_graph_hash,
558
+ community=community,
559
+ )
560
+ reused += 1
561
+ members_to_insert.extend((community_key, member) for member in community.members)
562
+
563
+ # Batched (perf-T4 G4): one ``executemany`` per call instead of N per-row
564
+ # round-trips (2,583 inserts for the live corpus collapse to one cursor
565
+ # batch). Runs inside the caller's open transaction so the bulk write is
566
+ # atomic with the surrounding membership replacement.
567
+ if members_to_insert:
568
+ with conn.cursor() as cur:
569
+ cur.executemany(
570
+ "INSERT INTO graph_community_members "
571
+ "(tenant_id, community_key, entity_id, member_rank, member_weight) "
572
+ "VALUES (%s, %s, %s, %s, %s)",
573
+ [
574
+ (
575
+ tenant,
576
+ community_key,
577
+ member.entity_id,
578
+ member.member_rank,
579
+ member.member_weight,
580
+ )
581
+ for community_key, member in members_to_insert
582
+ ],
583
+ )
584
+
585
+ return created, reused
586
+
587
+
588
+ def _insert_community(
589
+ conn: psycopg.Connection[Any],
590
+ tenant: str,
591
+ *,
592
+ community_key: str,
593
+ source_graph_hash: str,
594
+ community: DetectedCommunity,
595
+ ) -> None:
596
+ """INSERT a freshly-minted community row (NULL summary fields)."""
597
+ conn.execute(
598
+ "INSERT INTO graph_communities "
599
+ "(tenant_id, community_key, level, build_version, source_graph_hash, "
600
+ " members_hash, member_count, edge_count, total_weight) "
601
+ "VALUES (%s, %s, 0, %s, %s, %s, %s, %s, %s)",
602
+ (
603
+ tenant,
604
+ community_key,
605
+ BUILD_VERSION,
606
+ source_graph_hash,
607
+ community.members_hash,
608
+ community.member_count,
609
+ community.edge_count,
610
+ community.total_weight,
611
+ ),
612
+ )
613
+
614
+
615
+ def _update_reused_community(
616
+ conn: psycopg.Connection[Any],
617
+ tenant: str,
618
+ *,
619
+ community_key: str,
620
+ source_graph_hash: str,
621
+ community: DetectedCommunity,
622
+ ) -> None:
623
+ """UPDATE a reused community in place, PRESERVING its summary columns.
624
+
625
+ The summary/embedding columns are deliberately omitted from the SET clause
626
+ (the delta-gate; §17c Q3/Q10): a still-valid summary survives the rebuild and
627
+ a membership change is recorded only via ``members_hash`` for G3-c to act on.
628
+ """
629
+ conn.execute(
630
+ "UPDATE graph_communities SET "
631
+ "build_version = %s, source_graph_hash = %s, members_hash = %s, "
632
+ "member_count = %s, edge_count = %s, total_weight = %s, updated_at = NOW() "
633
+ "WHERE tenant_id = %s AND community_key = %s",
634
+ (
635
+ BUILD_VERSION,
636
+ source_graph_hash,
637
+ community.members_hash,
638
+ community.member_count,
639
+ community.edge_count,
640
+ community.total_weight,
641
+ tenant,
642
+ community_key,
643
+ ),
644
+ )