secondbrain-py 0.2.1__py3-none-any.whl

brain/migrations/001_init.sql

@@ -0,0 +1,43 @@
+CREATE EXTENSION IF NOT EXISTS vector;
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+CREATE TABLE sources (
+  id           UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  kind         TEXT NOT NULL,
+  external_id  TEXT,
+  metadata     JSONB NOT NULL DEFAULT '{}',
+  created_at   TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  UNIQUE (kind, external_id)
+);
+
+CREATE TABLE documents (
+  id            UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  source_id     UUID REFERENCES sources(id) ON DELETE CASCADE,
+  title         TEXT NOT NULL,
+  content       TEXT NOT NULL,
+  content_hash  TEXT NOT NULL UNIQUE,
+  content_type  TEXT NOT NULL,
+  source_path   TEXT,
+  tags          TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
+  metadata      JSONB NOT NULL DEFAULT '{}',
+  ingested_at   TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  tsv           tsvector GENERATED ALWAYS AS
+                  (to_tsvector('english', coalesce(title,'') || ' ' || coalesce(content,''))) STORED
+);
+CREATE INDEX documents_tsv_idx    ON documents USING GIN (tsv);
+CREATE INDEX documents_tags_idx   ON documents USING GIN (tags);
+CREATE INDEX documents_source_idx ON documents(source_id);
+
+CREATE TABLE chunks (
+  id           UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  document_id  UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  chunk_index  INT NOT NULL,
+  content      TEXT NOT NULL,
+  embedding    vector(1024) NOT NULL,
+  metadata     JSONB NOT NULL DEFAULT '{}',
+  tsv          tsvector GENERATED ALWAYS AS (to_tsvector('english', content)) STORED,
+  UNIQUE (document_id, chunk_index)
+);
+CREATE INDEX chunks_embedding_idx ON chunks USING hnsw (embedding vector_cosine_ops);
+CREATE INDEX chunks_tsv_idx       ON chunks USING GIN (tsv);
+CREATE INDEX chunks_document_idx  ON chunks(document_id);

brain/migrations/002_qwen3_embedding.sql

@@ -0,0 +1,17 @@
+-- Phase 2 of Voyage→Qwen3 swap. The user has explicitly approved discarding
+-- existing Voyage embeddings; chunks.content is preserved so Phase 3's
+-- `brain reembed` can repopulate the new column from the original chunk text.
+--
+-- The NOT NULL constraint is deferred to post-backfill — `brain reembed --finalize`
+-- applies it once 0 NULL rows remain. No vector index is created: pgvector 0.8.2
+-- caps both HNSW and IVFFlat at 2000 dims for `vector` and 4000 for `halfvec`,
+-- neither of which fits our native 4096-dim Qwen3 output. Sequential scan is
+-- acceptable at personal scale (~150ms @ 10K chunks).
+
+BEGIN;
+
+DROP INDEX IF EXISTS chunks_embedding_idx;
+ALTER TABLE chunks DROP COLUMN embedding;
+ALTER TABLE chunks ADD COLUMN embedding vector(4096);
+
+COMMIT;

brain/migrations/003_vault_model.sql

@@ -0,0 +1,41 @@
+-- Vault model phase 1 — schema additions for the headless-Obsidian vault.
+--
+-- Additive only: every existing row gets ``kind='ingested'`` via the column
+-- default (matches the legacy semantics where everything in the DB came from
+-- an upstream source). ``links`` and ``unresolved_links`` are new; nothing in
+-- prior migrations is altered or dropped.
+
+BEGIN;
+
+ALTER TABLE documents ADD COLUMN kind TEXT NOT NULL DEFAULT 'ingested'
+  CHECK (kind IN ('vault', 'ingested'));
+
+ALTER TABLE documents ADD COLUMN vault_path TEXT;
+CREATE UNIQUE INDEX documents_vault_path_idx
+  ON documents (vault_path) WHERE vault_path IS NOT NULL;
+CREATE INDEX documents_kind_idx ON documents (kind);
+
+CREATE TABLE links (
+  id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  src_document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  dst_document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  link_text       TEXT NOT NULL,
+  link_kind       TEXT NOT NULL CHECK (link_kind IN ('wiki', 'embed')),
+  display_text    TEXT,
+  UNIQUE (src_document_id, dst_document_id, link_text, link_kind)
+);
+CREATE INDEX links_src_idx ON links (src_document_id);
+CREATE INDEX links_dst_idx ON links (dst_document_id);
+
+CREATE TABLE unresolved_links (
+  id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  src_document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  link_text       TEXT NOT NULL,
+  link_kind       TEXT NOT NULL CHECK (link_kind IN ('wiki', 'embed')),
+  display_text    TEXT,
+  UNIQUE (src_document_id, link_text, link_kind)
+);
+CREATE INDEX unresolved_links_src_idx ON unresolved_links (src_document_id);
+CREATE INDEX unresolved_links_text_idx ON unresolved_links (link_text);
+
+COMMIT;

brain/migrations/004_relax_content_hash_uniqueness.sql

@@ -0,0 +1,18 @@
+-- Phase 2 follow-up: vault-tier notes can legitimately share body bytes.
+--
+-- Examples that hit the prior UNIQUE constraint and crashed sync:
+--   - Two empty notes (e.g. drafts the user just created from a template)
+--   - Two notes whose only body is "TBD" or a placeholder string
+--   - Daily notes for two days that happen to start with the same scaffold
+--
+-- Ingested-tier dedup is still desirable: re-ingesting the same Krisp call
+-- should idempotently no-op rather than producing two rows. Keep the unique
+-- constraint scoped to ``kind='ingested'`` via a partial index.
+
+BEGIN;
+
+ALTER TABLE documents DROP CONSTRAINT documents_content_hash_key;
+CREATE UNIQUE INDEX documents_content_hash_ingested_idx
+  ON documents (content_hash) WHERE kind = 'ingested';
+
+COMMIT;

brain/migrations/005_derived_links.sql

@@ -0,0 +1,67 @@
+-- Metadata-aware linker — schema for derived edges + person directory.
+--
+-- Additive only: nothing in prior migrations is altered or dropped. Three
+-- new tables back the metadata-aware linker pipeline:
+--
+--   * ``derived_links`` — sibling of ``links`` (which holds wiki-link edges
+--     only). Each row is an edge inferred from metadata overlap between two
+--     documents (e.g. shared Gmail thread, shared participant on the same
+--     day). Carries the rule that produced it, JSONB evidence, and a weight.
+--   * ``directory_entries`` — persistent name↔email directory used to bridge
+--     Krisp speaker labels (often a name) with Gmail headers (always an
+--     email). Built incrementally from already-stored Gmail headers, MCP
+--     calendar events, MCP contacts, and an optional ``people.yml`` override.
+--   * ``directory_refresh_state`` — high-water mark per source so calendar /
+--     contacts refreshes are incremental: each refresh fetches
+--     ``(last_refreshed_at, now]`` only.
+--
+-- Two CHECK constraints tighter than spec §5 (intentional, mirrors the
+--   `link_kind` pattern from migrations/003_vault_model.sql:23):
+--   - `rule` is enum-restricted; adding a 4th rule will require ALTER.
+--   - `weight` is range-restricted to [0, 1.0]; emphasized overrides above
+--      1.0 will require ALTER.
+
+BEGIN;
+
+-- Edges derived from metadata overlap; sibling of `links` (which holds
+-- wiki-link edges only).
+CREATE TABLE derived_links (
+  id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  src_document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  dst_document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  rule            TEXT NOT NULL CHECK (rule IN ('shared_thread', 'shared_participant', 'same_day_participant')),
+  evidence        JSONB NOT NULL DEFAULT '{}'::jsonb,
+  weight          REAL NOT NULL DEFAULT 1.0 CHECK (weight >= 0.0 AND weight <= 1.0),
+  CHECK (src_document_id <> dst_document_id),
+  UNIQUE (src_document_id, dst_document_id, rule)
+);
+CREATE INDEX derived_links_src_idx ON derived_links (src_document_id);
+CREATE INDEX derived_links_dst_idx ON derived_links (dst_document_id);
+CREATE INDEX derived_links_rule_idx ON derived_links (rule);
+
+-- Persistent name↔email directory used to bridge Krisp speaker labels
+-- (often a name) with Gmail headers (always an email). Built incrementally
+-- from already-stored Gmail headers, MCP calendar events, MCP contacts,
+-- and an optional override file.
+CREATE TABLE directory_entries (
+  id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  display_name    TEXT NOT NULL,             -- normalized lowercase
+  email           TEXT NOT NULL,             -- normalized lowercase
+  source          TEXT NOT NULL CHECK (source IN ('gmail', 'calendar', 'contacts', 'people_yml')),
+  occurrence_count INT NOT NULL DEFAULT 1,
+  first_seen_at   TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  last_seen_at    TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  UNIQUE (display_name, email, source)
+);
+CREATE INDEX directory_entries_name_idx ON directory_entries (display_name);
+CREATE INDEX directory_entries_email_idx ON directory_entries (email);
+
+-- High-water mark per source so calendar / contacts refreshes are
+-- incremental: each refresh fetches `(last_refreshed_at, now]` only.
+CREATE TABLE directory_refresh_state (
+  source              TEXT PRIMARY KEY CHECK (source IN ('gmail', 'calendar', 'contacts')),
+  last_refreshed_at   TIMESTAMPTZ NOT NULL,
+  records_seen        INT NOT NULL DEFAULT 0
+);
+
+COMMIT;

brain/migrations/006_dedup_file_by_source_path.sql

@@ -0,0 +1,25 @@
+-- File-based ingests dedup by ``source_path`` (in code), not ``content_hash``.
+--
+-- Why: when a source file's bytes change on disk and we re-ingest, the new
+-- content_hash differs from the prior row's, so the prior content_hash lookup
+-- missed and a NEW row was inserted — leaving the corpus with two documents
+-- for the same path. Triggering case: the markdown extractor fix in commit
+-- 85a6394 changed extracted-content bytes for every existing markdown doc, so
+-- every re-ingest would have silently duplicated.
+--
+-- Migration 004 introduced a partial UNIQUE index on ``content_hash`` scoped
+-- to ``kind='ingested'``. That worked while file ingests deduped by hash, but
+-- it conflicts with the new "two files with byte-identical content at
+-- different paths are separate documents" semantic — the second INSERT would
+-- fail. Tighten the partial predicate so the constraint only applies to
+-- stdin ingests (``source_path IS NULL``: krisp / slack / gmail), where we
+-- still want belt-and-suspenders protection against duplicate transcripts.
+
+BEGIN;
+
+DROP INDEX IF EXISTS documents_content_hash_ingested_idx;
+CREATE UNIQUE INDEX IF NOT EXISTS documents_content_hash_stdin_idx
+  ON documents (content_hash)
+  WHERE kind = 'ingested' AND source_path IS NULL;
+
+COMMIT;

brain/migrations/007_email_thread_and_draft.sql

@@ -0,0 +1,15 @@
+-- Migration 007: email-thread metadata + draft flag
+-- Additive only. Existing rows untouched.
+
+ALTER TABLE documents
+    ADD COLUMN IF NOT EXISTS thread_id        TEXT,
+    ADD COLUMN IF NOT EXISTS rfc_message_id   TEXT,
+    ADD COLUMN IF NOT EXISTS in_reply_to      TEXT,
+    ADD COLUMN IF NOT EXISTS sent_at          TIMESTAMPTZ,
+    ADD COLUMN IF NOT EXISTS participants     TEXT[],
+    ADD COLUMN IF NOT EXISTS duration_min     INTEGER,
+    ADD COLUMN IF NOT EXISTS draft            BOOLEAN NOT NULL DEFAULT FALSE;
+
+CREATE INDEX IF NOT EXISTS idx_documents_thread_id ON documents (thread_id) WHERE thread_id IS NOT NULL;
+CREATE INDEX IF NOT EXISTS idx_documents_sent_at   ON documents (sent_at DESC) WHERE sent_at IS NOT NULL;
+CREATE INDEX IF NOT EXISTS idx_documents_draft     ON documents (draft) WHERE draft = TRUE;

brain/migrations/008_gmail_thread_unique.sql

@@ -0,0 +1,11 @@
+-- Migration 008: enforce one merged-thread row per gmail thread_id.
+-- Additive only. Existing rows untouched (no thread_id collision today since
+-- the legacy ingest path keys on message_id; once P2.3 + P2.4 land, the
+-- collapsed corpus will have exactly one content_type='email_thread' row
+-- per thread and this index protects against accidental re-creation).
+
+CREATE UNIQUE INDEX IF NOT EXISTS uq_documents_gmail_thread
+    ON documents (thread_id)
+    WHERE kind = 'ingested'
+      AND thread_id IS NOT NULL
+      AND content_type = 'email_thread';

brain/migrations/009_chunks_weighted_tsv.sql

@@ -0,0 +1,28 @@
+-- Migration 009: title- and tag-weighted FTS for chunks.
+-- See docs/plans/2026-05-06-search-ranking-fix.md.
+--
+-- Adds denormalized title/tags/extras columns on chunks and recomputes
+-- chunks.tsv as a weighted multi-field tsvector so title/tag hits rank
+-- ahead of body hits. Additive: title_text/tags_text/search_extras are
+-- nullable; coalesce(...) inside the generated column keeps the tsv
+-- well-defined for rows that haven't been backfilled yet.
+--
+-- The DROP COLUMN tsv is technically destructive but the column is
+-- generated (no user data); it's rebuilt below from the new sources in
+-- the same migration. The GIN index is recreated against the new tsv.
+
+ALTER TABLE chunks ADD COLUMN IF NOT EXISTS title_text     TEXT;
+ALTER TABLE chunks ADD COLUMN IF NOT EXISTS tags_text      TEXT;
+ALTER TABLE chunks ADD COLUMN IF NOT EXISTS search_extras  TEXT;
+
+DROP INDEX IF EXISTS chunks_tsv_idx;
+ALTER TABLE chunks DROP COLUMN IF EXISTS tsv;
+
+ALTER TABLE chunks ADD COLUMN tsv tsvector GENERATED ALWAYS AS (
+    setweight(to_tsvector('english', coalesce(title_text,    '')), 'A') ||
+    setweight(to_tsvector('english', coalesce(tags_text,     '')), 'B') ||
+    setweight(to_tsvector('english', coalesce(content,       '')), 'C') ||
+    setweight(to_tsvector('english', coalesce(search_extras, '')), 'C')
+) STORED;
+
+CREATE INDEX chunks_tsv_idx ON chunks USING GIN (tsv);

brain/migrations/010_interactions.sql

@@ -0,0 +1,30 @@
+-- Wave Q1-C — Interactions table.
+-- Append-only feedback log. One row per user event (open / rate / pin / click).
+-- Q1-C populates from CLI (brain rate) and MCP (brain_show with
+-- originating_query). The wiki click surface lands in a later wave
+-- (source='wiki' is accepted at the SQL layer so the future wave is
+-- purely additive).
+-- Additive-only: nothing in prior migrations is altered.
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS interactions (
+  id           UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  document_id  UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  query        TEXT,
+  action       TEXT NOT NULL
+                 CHECK (action IN ('clicked', 'opened',
+                                   'rated_useful', 'rated_irrelevant',
+                                   'pinned')),
+  source       TEXT NOT NULL
+                 CHECK (source IN ('cli', 'mcp', 'wiki')),
+  session_id   UUID,   -- nullable: CLI ratings have no session
+  at           TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS interactions_document_at_idx ON interactions (document_id, at DESC);
+CREATE INDEX IF NOT EXISTS interactions_action_idx      ON interactions (action);
+CREATE INDEX IF NOT EXISTS interactions_session_idx     ON interactions (session_id)
+  WHERE session_id IS NOT NULL;
+
+COMMIT;

brain/migrations/011_documents_summary.sql

@@ -0,0 +1,23 @@
+-- Migration 011 — Wave Q1-D per-document auto-summary columns.
+--
+-- Additive only. Existing rows untouched. Pre-Q1-D rows keep
+-- ``summary IS NULL``; ``brain enrich --backfill`` populates them.
+--
+-- Three nullable columns + one partial index on ``summary IS NULL`` so the
+-- backfill iterator (``brain enrich --backfill``) can keyset-walk only the
+-- unenriched subset. The index stays tiny once enrichment has run over the
+-- corpus — only rows that genuinely failed enrichment (short content,
+-- Ollama unavailable, repeated parse failures) remain in it. Mirrors the
+-- ``idx_documents_thread_id`` shape from migration 007.
+
+BEGIN;
+
+ALTER TABLE documents
+    ADD COLUMN IF NOT EXISTS summary       TEXT,
+    ADD COLUMN IF NOT EXISTS summary_model TEXT,
+    ADD COLUMN IF NOT EXISTS summary_at    TIMESTAMPTZ;
+
+CREATE INDEX IF NOT EXISTS idx_documents_summary_null
+    ON documents (id) WHERE summary IS NULL;
+
+COMMIT;

brain/migrations/012_graphrag.sql

@@ -0,0 +1,171 @@
+-- Migration 012 — GraphRAG relational source-of-truth (wave G0, AGE pivot v5).
+--
+-- Additive only. This migration owns the RELATIONAL source-of-truth +
+-- pgvector side tables for the GraphRAG layer per spec
+-- docs/specs/2026-05-20-graphrag-design.md §5(a). The Apache AGE graph itself
+-- (graph `brain_graph`, vertex/edge labels, property indexes) is NOT created
+-- here — it is bootstrapped idempotently by ``brain init`` (a ``create_graph``
+-- function call, not frozen DDL) and its labels/indexes are owned by the G0-4
+-- GraphBackend. Keep this file pure SQL with no AGE/Cypher DDL.
+--
+--   graph_entities            — entity nodes (durable app identity; AGE vertex
+--                               IDs are internal and not the app identity).
+--                               People imported from the people pipeline;
+--                               concepts from the gated extractor.
+--   graph_entity_mentions     — SOURCE OF TRUTH: per-doc entity mentions
+--   graph_edge_contributions  — SOURCE OF TRUTH: per-doc raw window co-occurrence
+--   graph_relationships       — DERIVED aggregate edges (recomputed, never
+--                               blind-incremented); the SQL mirror of the AGE
+--                               CO_OCCURS edges (kept in sync by the same refresh)
+--   graph_index_state         — per-aspect incremental watermark (people vs
+--                               concepts re-index independently)
+--
+-- MULTI-TENANCY (spec §4 D9, §5): every table carries
+-- ``tenant_id TEXT NOT NULL DEFAULT 'default'`` and ``tenant_id`` is part of
+-- every PK / UNIQUE and every lookup index — every key below is tenant-scoped.
+-- ``graph_entities`` is keyed on the COMPOSITE ``(tenant_id, id)`` so the child
+-- tables can carry TENANT-SAFE composite FKs ``(tenant_id, <entity_col>) ->
+-- graph_entities(tenant_id, id)`` — a child row can only ever reference an
+-- entity in its OWN tenant. ``id`` remains a globally-unique UUID (it is the AGE
+-- ``Entity.entity_uuid`` vertex property); the composite key just makes the
+-- relational key tenant-inclusive. Single-user local deployments use the fixed
+-- default tenant ``'default'`` so the local experience is unchanged.
+--
+-- Per-document rows are the only source of truth; aggregates are derived by a
+-- full refresh (spec §7). Undirected edges are canonicalized ``src_id < dst_id``
+-- and CHECK-enforced. ``graph_relationships.weight`` is normalized lift in
+-- (0, 1] (CHECK-enforced, NOT NULL with NO default — callers always compute it).
+-- Count columns are CHECK-enforced non-negative. Embedding columns ship as
+-- ``vector(1024)`` NULLABLE; the generalized dim reconciliation (G0b) resizes
+-- ``graph_entities.embedding`` to the active embedder's dim and a later wave
+-- embeds it. HNSW is intentionally skipped here (small row counts →
+-- sequential scan; spec §5 note).
+--
+-- v5.1 note: the paused g0a created an uncommitted, pre-tenant/pre-AGE version
+-- of this file; since it was never committed/shipped, v5.1 rewrites it in place
+-- to this tenantized schema (no separate migration).
+--
+-- Communities tables arrive in 013 (G3); the interaction-logging change in
+-- 014 (G4).
+
+BEGIN;
+
+-- Entity nodes. Person rows imported from the people pipeline; concepts from
+-- the extractor. ``doc_count`` is DERIVED from mentions (refreshed, never
+-- authoritative on write). Durable app identity lives in ``id``; the AGE
+-- ``Entity`` vertex carries ``entity_uuid = id`` as a property for joins back.
+-- COMPOSITE PK ``(tenant_id, id)`` makes the key tenant-inclusive and is the
+-- target of the child tables' tenant-safe FKs.
+CREATE TABLE IF NOT EXISTS graph_entities (
+    id            UUID NOT NULL DEFAULT gen_random_uuid(),
+    tenant_id     TEXT NOT NULL DEFAULT 'default',
+    entity_type   TEXT NOT NULL,            -- 'person'|'org'|'project'|'topic'|'tool'
+    name          TEXT NOT NULL,
+    canonical_key TEXT NOT NULL,            -- dedup key (resolved person-key | lower(name))
+    description   TEXT,
+    embedding     vector(1024),             -- dim-aware; nullable until reconciled
+    doc_count     INTEGER NOT NULL DEFAULT 0,   -- DERIVED from mentions (refreshed)
+    properties    JSONB NOT NULL DEFAULT '{}',
+    created_at    TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at    TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    PRIMARY KEY (tenant_id, id),
+    CONSTRAINT graph_entities_type_chk
+        CHECK (entity_type IN ('person', 'org', 'project', 'topic', 'tool')),
+    CONSTRAINT graph_entities_doc_count_chk CHECK (doc_count >= 0),
+    -- Tenant-scoped dedup: the same canonical_key may exist independently per tenant.
+    CONSTRAINT uq_graph_entities UNIQUE (tenant_id, entity_type, canonical_key)
+);
+CREATE INDEX IF NOT EXISTS idx_graph_entities_type
+    ON graph_entities (tenant_id, entity_type);
+
+-- SOURCE OF TRUTH: per-doc entity mentions. Re-ingest deletes+reinserts this
+-- doc's rows. Tenant-safe composite FK ``(tenant_id, entity_id)`` guarantees the
+-- referenced entity is in the SAME tenant. The document_id FK stays single-col
+-- (documents is not tenantized in G0).
+CREATE TABLE IF NOT EXISTS graph_entity_mentions (
+    tenant_id     TEXT NOT NULL DEFAULT 'default',
+    entity_id     UUID NOT NULL,
+    document_id   UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+    mention_count INTEGER NOT NULL DEFAULT 1,
+    source        TEXT NOT NULL,            -- 'people' | 'extractor:<model>@<ver>'
+    PRIMARY KEY (tenant_id, entity_id, document_id),
+    CONSTRAINT gem_mention_count_chk CHECK (mention_count >= 0),
+    CONSTRAINT fk_gem_entity
+        FOREIGN KEY (tenant_id, entity_id)
+        REFERENCES graph_entities (tenant_id, id) ON DELETE CASCADE
+);
+CREATE INDEX IF NOT EXISTS idx_gem_document
+    ON graph_entity_mentions (tenant_id, document_id);
+
+-- SOURCE OF TRUTH: per-doc edge contributions (raw window co-occurrence over
+-- raw text). RAW counts only — no generic suppression applied here (suppression
+-- is derive/query-time). Canonical src<dst. Both endpoints carry tenant-safe
+-- composite FKs.
+CREATE TABLE IF NOT EXISTS graph_edge_contributions (
+    tenant_id     TEXT NOT NULL DEFAULT 'default',
+    document_id   UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+    src_id        UUID NOT NULL,
+    dst_id        UUID NOT NULL,
+    cooccur_count INTEGER NOT NULL DEFAULT 1,
+    PRIMARY KEY (tenant_id, document_id, src_id, dst_id),
+    CONSTRAINT gec_canonical CHECK (src_id < dst_id),
+    CONSTRAINT gec_cooccur_count_chk CHECK (cooccur_count >= 0),
+    CONSTRAINT fk_gec_src
+        FOREIGN KEY (tenant_id, src_id)
+        REFERENCES graph_entities (tenant_id, id) ON DELETE CASCADE,
+    CONSTRAINT fk_gec_dst
+        FOREIGN KEY (tenant_id, dst_id)
+        REFERENCES graph_entities (tenant_id, id) ON DELETE CASCADE
+);
+CREATE INDEX IF NOT EXISTS idx_gec_src ON graph_edge_contributions (tenant_id, src_id);
+CREATE INDEX IF NOT EXISTS idx_gec_dst ON graph_edge_contributions (tenant_id, dst_id);
+
+-- DERIVED aggregate edges (recomputed from contributions; never
+-- blind-incremented). SQL counterpart of the AGE CO_OCCURS edges (kept in sync
+-- by the same refresh) and backs evidence/ranking. ``weight`` is normalized
+-- lift in (0, 1] — NOT NULL with NO default (callers always compute it; 0 would
+-- violate the range). Both endpoints carry tenant-safe composite FKs.
+CREATE TABLE IF NOT EXISTS graph_relationships (
+    tenant_id  TEXT NOT NULL DEFAULT 'default',
+    src_id     UUID NOT NULL,
+    dst_id     UUID NOT NULL,
+    rel_type   TEXT NOT NULL DEFAULT 'co_occurs',
+    weight     REAL NOT NULL,               -- normalized lift in (0,1] (derived/recomputed)
+    co_count   INTEGER NOT NULL DEFAULT 0,  -- SUM(contributions.cooccur_count)
+    doc_count  INTEGER NOT NULL DEFAULT 0,  -- COUNT(DISTINCT document_id)
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    PRIMARY KEY (tenant_id, src_id, dst_id, rel_type),
+    CONSTRAINT grel_canonical CHECK (src_id < dst_id),
+    CONSTRAINT grel_weight_chk CHECK (weight > 0 AND weight <= 1),
+    CONSTRAINT grel_co_count_chk CHECK (co_count >= 0),
+    CONSTRAINT grel_doc_count_chk CHECK (doc_count >= 0),
+    CONSTRAINT fk_grel_src
+        FOREIGN KEY (tenant_id, src_id)
+        REFERENCES graph_entities (tenant_id, id) ON DELETE CASCADE,
+    CONSTRAINT fk_grel_dst
+        FOREIGN KEY (tenant_id, dst_id)
+        REFERENCES graph_entities (tenant_id, id) ON DELETE CASCADE
+);
+CREATE INDEX IF NOT EXISTS idx_grel_src ON graph_relationships (tenant_id, src_id);
+CREATE INDEX IF NOT EXISTS idx_grel_dst ON graph_relationships (tenant_id, dst_id);
+
+-- Incremental watermark, PER ASPECT (people vs concepts re-index
+-- independently). An aspect is re-indexed when ANY of its tracked input
+-- fingerprints changes (content_hash + inputs_hash + extractor_ver +
+-- suppress_ver; spec §7). ``aspect`` is constrained to the two known aspects:
+-- 'people' (G1) and 'concepts' (G2).
+CREATE TABLE IF NOT EXISTS graph_index_state (
+    tenant_id      TEXT NOT NULL DEFAULT 'default',
+    document_id    UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+    aspect         TEXT NOT NULL,           -- 'people' | 'concepts'
+    content_hash   TEXT NOT NULL,           -- body text fingerprint
+    inputs_hash    TEXT NOT NULL,           -- aspect inputs: people→participants+directory+owner-key
+                                            --   versions; concepts→content+window/cap config
+    extractor_ver  TEXT NOT NULL,           -- extractor model@version; bump forces re-extract
+    suppress_ver   TEXT NOT NULL DEFAULT '',-- suppression/weighting config version (derive-time)
+    indexed_at     TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    PRIMARY KEY (tenant_id, document_id, aspect),
+    CONSTRAINT gis_aspect_chk CHECK (aspect IN ('people', 'concepts'))
+);
+
+COMMIT;

brain/migrations/013_graphrag_communities.sql

@@ -0,0 +1,125 @@
+-- Migration 013 — GraphRAG global communities (wave G3, AGE pivot v5).
+--
+-- Additive only. This migration owns the RELATIONAL community tables for the
+-- LazyGraphRAG-style global mode per spec docs/specs/2026-05-20-graphrag-design.md
+-- §17c Q1. Communities are RELATIONAL-ONLY (§17c Q2): there is NO AGE
+-- ``Community`` vertex label and no membership edges in AGE — the graph stays
+-- ``Entity``/``Document`` + ``MENTIONED_IN``/``CO_OCCURS``. Community summaries,
+-- memberships, and embeddings live here in SQL exactly like the embeddings +
+-- derived aggregates. Keep this file pure SQL with no AGE/Cypher DDL.
+--
+--   graph_communities         — one row per detected community (networkx Louvain
+--                               over the tenant entity graph). Carries the
+--                               stable ``community_key`` identity, the dirty
+--                               fingerprint (``source_graph_hash``), the
+--                               per-community identity hash (``members_hash``),
+--                               aggregate stats, and the lazily-built (eager at
+--                               community build/refresh — §17c Q10) summary +
+--                               summary embedding/tsv for the global retrieval
+--                               RRF (§17c Q4).
+--   graph_community_members   — community ↔ entity membership (per-community
+--                               rank + weight). Tenant-safe composite FKs to
+--                               BOTH graph_communities and graph_entities.
+--
+-- SINGLE LEVEL ONLY (§17c Q1 / §15): ``level`` is present but pinned to 0 via a
+-- CHECK; hierarchical multi-level communities stay deferred (YAGNI).
+--
+-- MULTI-TENANCY (spec §4 D9, §5): every table carries
+-- ``tenant_id TEXT NOT NULL DEFAULT 'default'`` and ``tenant_id`` is part of
+-- every PK / UNIQUE and every lookup index — every key below is tenant-scoped.
+-- The membership table carries TENANT-SAFE composite FKs
+-- ``(tenant_id, community_key) -> graph_communities(tenant_id, community_key)``
+-- and ``(tenant_id, entity_id) -> graph_entities(tenant_id, id)`` so a member
+-- row can only ever reference a community / entity in its OWN tenant. Mirrors
+-- the 012 conventions exactly.
+--
+-- DIRTY FINGERPRINT (§17c Q1/Q3): ``source_graph_hash`` is a tenant-graph edge
+-- hash over the ordered ``graph_relationships``; ``communities build`` skips
+-- when the stored ``(build_version, source_graph_hash)`` already matches.
+-- ``members_hash`` is per-community identity only (Jaccard stable-identity).
+--
+-- EMBEDDINGS: ``summary_embedding`` ships as ``vector(1024)`` NULLABLE. The
+-- generalized dim reconciliation (G0b — allowlisted in
+-- ``brain.embedding_targets`` as ``graph_communities.summary_embedding``)
+-- resizes it to the active embedder's dim, and the conditional HNSW is owned by
+-- the finalize-time machinery (``brain.queries.finalize_embedding_index``), NOT
+-- hardcoded here — exactly as 012 defers ``graph_entities.embedding`` (small row
+-- counts → sequential cosine scan until finalize). The ``summary_tsv`` GENERATED
+-- column backs the FTS leg of the global RRF (§17c Q4); it uses the immutable
+-- 2-arg ``to_tsvector('english', …)`` form so it is valid in a STORED column
+-- (matches 009's chunks.tsv style).
+
+BEGIN;
+
+-- One row per detected community. ``community_key`` is the durable, stable
+-- identity preserved across rebuilds by Jaccard matching (§17c Q3/Q7).
+-- ``source_graph_hash`` + ``members_hash`` have NO default — the detector always
+-- computes them. Aggregate counts default to 0 and are CHECK-enforced
+-- non-negative (mirrors 012). PK ``(tenant_id, community_key)``; UNIQUE
+-- ``(tenant_id, level, members_hash)`` is the per-community identity guard.
+CREATE TABLE IF NOT EXISTS graph_communities (
+    tenant_id         TEXT NOT NULL DEFAULT 'default',
+    community_key     UUID NOT NULL DEFAULT gen_random_uuid(),
+    level             INTEGER NOT NULL DEFAULT 0,   -- single-level only (§15)
+    build_version     TEXT NOT NULL DEFAULT 'networkx-louvain-v1',
+    source_graph_hash TEXT NOT NULL,                -- dirty fingerprint (§17c Q3)
+    members_hash      TEXT NOT NULL,                -- per-community identity hash
+    member_count      INTEGER NOT NULL DEFAULT 0,
+    edge_count        INTEGER NOT NULL DEFAULT 0,
+    total_weight      REAL NOT NULL DEFAULT 0,
+    summary           TEXT,                          -- lazy/eager at build (§17c Q10)
+    summary_model     TEXT,
+    summary_at        TIMESTAMPTZ,
+    summary_embedding vector(1024),                  -- dim-aware; nullable; HNSW deferred
+    summary_tsv       tsvector GENERATED ALWAYS AS (
+                          to_tsvector('english', coalesce(summary, ''))
+                      ) STORED,
+    created_at        TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at        TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    PRIMARY KEY (tenant_id, community_key),
+    CONSTRAINT gc_level_chk CHECK (level = 0),
+    CONSTRAINT gc_member_count_chk CHECK (member_count >= 0),
+    CONSTRAINT gc_edge_count_chk CHECK (edge_count >= 0),
+    CONSTRAINT gc_total_weight_chk CHECK (total_weight >= 0),
+    -- Per-community identity is tenant-scoped: the same members_hash may exist
+    -- independently per tenant (and, once multi-level lands, per level).
+    CONSTRAINT uq_graph_communities UNIQUE (tenant_id, level, members_hash)
+);
+-- Global retrieval scans a tenant's level-0 communities; the PK leads with
+-- tenant_id, but the explicit (tenant_id, level) index documents the
+-- candidate-set access pattern and survives the multi-level deferral.
+CREATE INDEX IF NOT EXISTS idx_graph_communities_tenant_level
+    ON graph_communities (tenant_id, level);
+-- FTS leg of the global RRF (§17c Q4).
+CREATE INDEX IF NOT EXISTS idx_graph_communities_summary_tsv
+    ON graph_communities USING GIN (summary_tsv);
+
+-- Community ↔ entity membership. Re-detection deletes+reinserts a tenant's
+-- rows. Tenant-safe composite FKs guarantee both the community and the entity
+-- live in the SAME tenant. ``member_rank`` / ``member_weight`` order entities
+-- within a community (e.g. by degree / centrality); both CHECK-enforced
+-- non-negative.
+CREATE TABLE IF NOT EXISTS graph_community_members (
+    tenant_id     TEXT NOT NULL DEFAULT 'default',
+    community_key UUID NOT NULL,
+    entity_id     UUID NOT NULL,
+    member_rank   INTEGER NOT NULL DEFAULT 0,
+    member_weight REAL NOT NULL DEFAULT 0,
+    PRIMARY KEY (tenant_id, community_key, entity_id),
+    CONSTRAINT gcm_member_rank_chk CHECK (member_rank >= 0),
+    CONSTRAINT gcm_member_weight_chk CHECK (member_weight >= 0),
+    CONSTRAINT fk_gcm_community
+        FOREIGN KEY (tenant_id, community_key)
+        REFERENCES graph_communities (tenant_id, community_key) ON DELETE CASCADE,
+    CONSTRAINT fk_gcm_entity
+        FOREIGN KEY (tenant_id, entity_id)
+        REFERENCES graph_entities (tenant_id, id) ON DELETE CASCADE
+);
+-- Reverse lookup (which communities an entity belongs to) + makes the
+-- graph_entities-delete cascade over this FK fast (Postgres does not
+-- auto-index FK referencing columns). The (tenant_id, community_key) FK is
+-- already covered by the PK's leading columns.
+CREATE INDEX IF NOT EXISTS idx_gcm_entity
+    ON graph_community_members (tenant_id, entity_id);
+
+COMMIT;