secondbrain-py 0.2.1__py3-none-any.whl

brain/review/queries.py

@@ -0,0 +1,436 @@
+"""SQL helpers for ``brain review scan`` (Plan 03 — contradiction + staleness).
+
+One reason to change: the shape of ``elicitation_gaps`` or the GraphRAG tables
+this module reads (``graph_entities`` / ``graph_entity_mentions`` /
+``documents`` / ``chunks``). All queries are parameterized and tenant-scoped
+where a tenant column exists. The pure scan logic lives in :mod:`brain.review.scans`.
+"""
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any
+
+import psycopg
+
+# Review-finding signal kinds owned by this surface. ``brain review list`` /
+# ``dismiss`` only ever touch these two; the other ``elicitation_gaps`` kinds
+# (delta / orphan / contradiction-from-elicit / user_flagged) belong to
+# ``brain elicit``.
+REVIEW_SIGNAL_KINDS: tuple[str, ...] = ("contradiction", "stale")
+
+# Document content types that are historical records, not living claims — never
+# eligible as staleness candidates (a transcript of a past meeting does not go
+# "stale" the way a compensation-range note does).
+_STALE_EXCLUDED_CONTENT_TYPES: tuple[str, ...] = ("transcript", "krisp_action_items")
+
+
+@dataclass(frozen=True)
+class EntityCandidate:
+    """One entity surviving the conflict-scan graph prefilter (Step 1)."""
+
+    canonical_key: str
+    name: str
+    entity_type: str
+    doc_ids: list[str]
+
+
+@dataclass(frozen=True)
+class StaleCandidate:
+    """One aged document eligible for the staleness scan (Step 1)."""
+
+    doc_id: str
+    title: str
+    age_days: int
+    content_type: str
+
+
+@dataclass(frozen=True)
+class SupersedingDoc:
+    """A newer document sharing an entity with a stale candidate (Step 2)."""
+
+    doc_id: str
+    title: str
+
+
+@dataclass(frozen=True)
+class QueueRow:
+    """One row of the ``brain review list`` queue read-back."""
+
+    id: str
+    signal_kind: str
+    target_type: str
+    target_id: str
+    score: float
+    evidence_ids: list[str]
+    rationale: str
+    status: str
+
+
+def iter_entities_for_conflict_scan(
+    conn: psycopg.Connection[Any],
+    *,
+    tenant_id: str,
+    min_docs: int,
+    limit: int,
+) -> list[EntityCandidate]:
+    """Graph prefilter: entities mentioned in >= ``min_docs`` summarized docs.
+
+    Only non-draft documents with a non-null ``summary`` count toward the
+    threshold (the summary is the evidence text fed to the LLM). Ordered by
+    distinct-doc count DESC and capped at ``limit`` so the LLM-call budget is
+    bounded. Pure SQL — no LLM, no embedder.
+    """
+    rows = conn.execute(
+        """
+        SELECT ge.canonical_key, ge.name, ge.entity_type,
+               array_agg(DISTINCT gem.document_id::text) AS doc_ids
+        FROM graph_entity_mentions gem
+        JOIN graph_entities ge
+          ON ge.id = gem.entity_id AND ge.tenant_id = gem.tenant_id
+        JOIN documents d ON d.id = gem.document_id
+        WHERE gem.tenant_id = %s
+          AND d.summary IS NOT NULL
+          AND d.draft IS NOT TRUE
+        GROUP BY ge.id, ge.canonical_key, ge.name, ge.entity_type
+        HAVING count(DISTINCT gem.document_id) >= %s
+        ORDER BY count(DISTINCT gem.document_id) DESC
+        LIMIT %s
+        """,
+        (tenant_id, min_docs, limit),
+    ).fetchall()
+    return [
+        EntityCandidate(
+            canonical_key=ckey,
+            name=name,
+            entity_type=etype,
+            doc_ids=list(doc_ids),
+        )
+        for (ckey, name, etype, doc_ids) in rows
+    ]
+
+
+def count_conflict_docs_missing_summary(
+    conn: psycopg.Connection[Any], *, tenant_id: str
+) -> int:
+    """Count non-draft docs referenced by an entity that still lack a summary.
+
+    Surfaced by ``brain review scan --conflicts --dry-run`` to nudge the user
+    toward ``brain enrich --backfill`` — these documents are silently excluded
+    from the graph prefilter above.
+    """
+    row = conn.execute(
+        """
+        SELECT count(DISTINCT d.id)
+        FROM graph_entity_mentions gem
+        JOIN documents d ON d.id = gem.document_id
+        WHERE gem.tenant_id = %s
+          AND d.draft IS NOT TRUE
+          AND d.summary IS NULL
+        """,
+        (tenant_id,),
+    ).fetchone()
+    return int(row[0]) if row else 0
+
+
+def iter_docs_for_staleness_scan(
+    conn: psycopg.Connection[Any],
+    *,
+    tenant_id: str,
+    stale_age_days: int,
+    limit: int,
+) -> list[StaleCandidate]:
+    """Age candidates: non-draft, summarized, non-transcript docs older than N days.
+
+    ``documents`` has no tenant column, so the scan is tenant-scoped via an
+    ``EXISTS`` on ``graph_entity_mentions``: only docs that participate in this
+    tenant's entity graph are candidates. That is exactly the set that could
+    ever yield a finding — the superseding-doc lookup (Step 2) requires a
+    tenant-shared entity — so the ``EXISTS`` both enforces tenant isolation and
+    skips docs that can never supersede. Oldest first; capped at ``limit``.
+    """
+    rows = conn.execute(
+        """
+        SELECT d.id::text, d.title,
+               (now()::date - d.ingested_at::date) AS age_days,
+               d.content_type
+        FROM documents d
+        WHERE d.ingested_at < now() - make_interval(days => %s)
+          AND d.content_type <> ALL(%s)
+          AND d.draft IS NOT TRUE
+          AND d.summary IS NOT NULL
+          AND EXISTS (
+              SELECT 1 FROM graph_entity_mentions gem
+              WHERE gem.document_id = d.id AND gem.tenant_id = %s
+          )
+        ORDER BY d.ingested_at ASC
+        LIMIT %s
+        """,
+        (stale_age_days, list(_STALE_EXCLUDED_CONTENT_TYPES), tenant_id, limit),
+    ).fetchall()
+    return [
+        StaleCandidate(
+            doc_id=doc_id,
+            title=title,
+            age_days=int(age_days),
+            content_type=content_type,
+        )
+        for (doc_id, title, age_days, content_type) in rows
+    ]
+
+
+def count_stale_docs_missing_summary(
+    conn: psycopg.Connection[Any], *, tenant_id: str, stale_age_days: int
+) -> int:
+    """Count aged non-draft, non-transcript docs that still lack a summary.
+
+    Tenant-scoped via the same ``graph_entity_mentions`` ``EXISTS`` as
+    :func:`iter_docs_for_staleness_scan`, so the
+    ``brain review scan --stale --dry-run`` nudge counts only docs in the active
+    tenant's graph that never enter the staleness pipeline for want of a summary.
+    """
+    row = conn.execute(
+        """
+        SELECT count(*)
+        FROM documents d
+        WHERE d.ingested_at < now() - make_interval(days => %s)
+          AND d.content_type <> ALL(%s)
+          AND d.draft IS NOT TRUE
+          AND d.summary IS NULL
+          AND EXISTS (
+              SELECT 1 FROM graph_entity_mentions gem
+              WHERE gem.document_id = d.id AND gem.tenant_id = %s
+          )
+        """,
+        (stale_age_days, list(_STALE_EXCLUDED_CONTENT_TYPES), tenant_id),
+    ).fetchone()
+    return int(row[0]) if row else 0
+
+
+def fetch_superseding_docs(
+    conn: psycopg.Connection[Any],
+    *,
+    tenant_id: str,
+    doc_id: str,
+    window_days: int,
+) -> list[SupersedingDoc]:
+    """Newer non-draft docs sharing >= 1 entity with ``doc_id``, ingested recently.
+
+    A superseding candidate must (a) share at least one graph entity with the
+    stale doc, (b) be ingested within ``window_days`` of now, and (c) not be the
+    stale doc itself. Tenant-scoped via ``graph_entity_mentions``.
+    """
+    rows = conn.execute(
+        """
+        SELECT DISTINCT d2.id::text, d2.title
+        FROM graph_entity_mentions m1
+        JOIN graph_entity_mentions m2
+          ON m2.tenant_id = m1.tenant_id AND m2.entity_id = m1.entity_id
+        JOIN documents d2 ON d2.id = m2.document_id
+        WHERE m1.tenant_id = %s
+          AND m1.document_id = %s
+          AND m2.document_id <> %s
+          AND d2.ingested_at >= now() - make_interval(days => %s)
+          AND d2.draft IS NOT TRUE
+        """,
+        (tenant_id, doc_id, doc_id, window_days),
+    ).fetchall()
+    return [SupersedingDoc(doc_id=did, title=title) for (did, title) in rows]
+
+
+def fetch_best_chunk_embeddings(
+    conn: psycopg.Connection[Any], *, document_ids: Sequence[str]
+) -> dict[str, list[float]]:
+    """Map each document id to its lead-chunk embedding (``chunk_index`` 0).
+
+    ``chunks`` has no ``created_at`` column, so the lead chunk is selected by
+    ``ORDER BY document_id, chunk_index`` under ``DISTINCT ON (document_id)``.
+    Documents whose chunks are all NULL-embedded are absent from the result.
+    Vectors are returned as plain ``list[float]`` (the ``pgvector`` adapter
+    yields a numpy array; we copy it out so the pure-Python cosine never depends
+    on numpy).
+    """
+    if not document_ids:
+        return {}
+    rows = conn.execute(
+        """
+        SELECT DISTINCT ON (document_id) document_id::text, embedding
+        FROM chunks
+        WHERE document_id = ANY(%s::uuid[])
+          AND embedding IS NOT NULL
+        ORDER BY document_id, chunk_index
+        """,
+        (list(document_ids),),
+    ).fetchall()
+    return {doc_id: [float(x) for x in vec] for (doc_id, vec) in rows}
+
+
+def fetch_doc_summaries(
+    conn: psycopg.Connection[Any], *, document_ids: Sequence[str]
+) -> dict[str, str]:
+    """Map each document id to its non-null ``summary`` text.
+
+    Documents with a NULL summary are omitted, so a caller can treat "absent"
+    and "no summary" identically. Used to assemble the LLM evidence pair.
+    """
+    if not document_ids:
+        return {}
+    rows = conn.execute(
+        """
+        SELECT id::text, summary
+        FROM documents
+        WHERE id = ANY(%s::uuid[]) AND summary IS NOT NULL
+        """,
+        (list(document_ids),),
+    ).fetchall()
+    return {doc_id: summary for (doc_id, summary) in rows}
+
+
+def existing_finding_statuses(
+    conn: psycopg.Connection[Any], *, tenant_id: str, signal_kind: str
+) -> dict[str, str]:
+    """Map ``target_id`` -> ``status`` for every non-resolved finding of a kind.
+
+    Backs the idempotency check (Step 2 of both scans): a target already
+    surfaced / snoozed / dismissed is skipped (never re-adjudicated); only a
+    ``resolved`` target (absent here) or a brand-new target is rescanned. The
+    partial unique index on ``elicitation_gaps`` guarantees at most one
+    non-resolved row per ``(tenant_id, signal_kind, target_id)``.
+    """
+    rows = conn.execute(
+        """
+        SELECT target_id, status
+        FROM elicitation_gaps
+        WHERE tenant_id = %s AND signal_kind = %s AND status <> 'resolved'
+        """,
+        (tenant_id, signal_kind),
+    ).fetchall()
+    return {target_id: status for (target_id, status) in rows}
+
+
+def upsert_review_finding(
+    conn: psycopg.Connection[Any],
+    *,
+    tenant_id: str,
+    signal_kind: str,
+    target_type: str,
+    target_id: str,
+    score: float,
+    evidence_ids: Sequence[str],
+    rationale: str,
+) -> None:
+    """Upsert one finding, never overwriting a user-dismissed row.
+
+    The partial unique index ``WHERE status <> 'resolved'`` covers dismissed
+    rows, so a naive upsert would clobber them. The ``DO UPDATE ... WHERE
+    elicitation_gaps.status IN ('surfaced','snoozed')`` guard detects the
+    conflict (dismissed != resolved → in the index) but skips the UPDATE for a
+    dismissed row, leaving the user's dismissal intact.
+    """
+    conn.execute(
+        """
+        INSERT INTO elicitation_gaps
+            (tenant_id, signal_kind, target_type, target_id,
+             score, evidence_ids, rationale)
+        VALUES (%s, %s, %s, %s, %s, %s, %s)
+        ON CONFLICT (tenant_id, signal_kind, target_id) WHERE status <> 'resolved'
+        DO UPDATE SET
+            score        = EXCLUDED.score,
+            evidence_ids = EXCLUDED.evidence_ids,
+            rationale    = EXCLUDED.rationale,
+            updated_at   = now()
+        WHERE elicitation_gaps.status IN ('surfaced', 'snoozed')
+        """,
+        (
+            tenant_id,
+            signal_kind,
+            target_type,
+            target_id,
+            score,
+            list(evidence_ids),
+            rationale,
+        ),
+    )
+
+
+def list_review_queue(
+    conn: psycopg.Connection[Any],
+    *,
+    tenant_id: str,
+    signal_kinds: Sequence[str],
+    limit: int,
+) -> list[QueueRow]:
+    """Read the open review queue (surfaced / snoozed) for the given kinds.
+
+    Scoped to ``signal_kinds`` (a subset of :data:`REVIEW_SIGNAL_KINDS`) so the
+    review queue stays independent of the ``brain elicit`` gap queue. Ordered by
+    score DESC. Snoozed rows whose ``snoozed_until`` is still in the future are
+    held back, mirroring ``brain elicit list``.
+    """
+    rows = conn.execute(
+        """
+        SELECT id::text, signal_kind, target_type, target_id,
+               score, evidence_ids, rationale, status
+        FROM elicitation_gaps
+        WHERE tenant_id = %s
+          AND signal_kind = ANY(%s)
+          AND status IN ('surfaced', 'snoozed')
+          AND (snoozed_until IS NULL OR snoozed_until < now())
+        ORDER BY score DESC
+        LIMIT %s
+        """,
+        (tenant_id, list(signal_kinds), limit),
+    ).fetchall()
+    return [
+        QueueRow(
+            id=r[0],
+            signal_kind=r[1],
+            target_type=r[2],
+            target_id=r[3],
+            score=float(r[4]),
+            evidence_ids=list(r[5]),
+            rationale=r[6],
+            status=r[7],
+        )
+        for r in rows
+    ]
+
+
+def dismiss_review_finding(
+    conn: psycopg.Connection[Any], *, tenant_id: str, id_prefix: str
+) -> str:
+    """Set ``status='dismissed'`` on the review finding matching ``id_prefix``.
+
+    Resolves ``id_prefix`` against open / dismissed review findings only
+    (``signal_kind`` in :data:`REVIEW_SIGNAL_KINDS`, ``status <> 'resolved'``).
+    Idempotent: dismissing an already-dismissed finding is a no-op that returns
+    its id. Returns the full finding id. Raises :class:`ValueError` when the
+    prefix matches no finding or is ambiguous — the CLI / MCP layer maps that to
+    a user-facing error.
+    """
+    matches = conn.execute(
+        """
+        SELECT id::text
+        FROM elicitation_gaps
+        WHERE tenant_id = %s
+          AND signal_kind = ANY(%s)
+          AND status <> 'resolved'
+          AND id::text LIKE %s
+        LIMIT 2
+        """,
+        (tenant_id, list(REVIEW_SIGNAL_KINDS), f"{id_prefix}%"),
+    ).fetchall()
+    if not matches:
+        raise ValueError(f"no review finding matches prefix {id_prefix!r}")
+    if len(matches) > 1:
+        raise ValueError(f"review finding prefix {id_prefix!r} is ambiguous")
+    finding_id = str(matches[0][0])
+    conn.execute(
+        """
+        UPDATE elicitation_gaps
+        SET status = 'dismissed', updated_at = now()
+        WHERE id = %s::uuid
+        """,
+        (finding_id,),
+    )
+    return finding_id

brain/review/render.py

@@ -0,0 +1,196 @@
+"""Pure renderers for a :class:`~brain.review.weekly.WeeklyReport`.
+
+``render_weekly_md`` produces the vault page (YAML frontmatter + Markdown body);
+``render_weekly_rich`` produces a compact terminal summary. Both are pure: the
+same ``WeeklyReport`` always yields byte-identical output, which is what makes
+the vault write idempotent (re-running a week overwrites with identical bytes).
+The only collaborators are the deterministic
+:func:`brain.vault.frontmatter.dump_frontmatter` and the wiki-link path helpers.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+from ..vault.frontmatter import dump_frontmatter
+from ..vault.paths import safe_wikilink_alias, strip_md_extension
+from .weekly import ThemeBlock, WeeklyReport
+
+_EMPTY_THEMES = "_No themes detected this week._"
+_EMPTY_ACTIVITY = "_No documents interacted with this week._"
+_EMPTY_LOOPS = "_No open loops._"
+_EMPTY_INGESTED = "_No new captures this week._"
+_EMPTY_PEOPLE = "_No key people identified._"
+
+
+def _doc_link(doc_id: str, title: str, vault_paths: dict[str, str]) -> str:
+    """Render a doc reference as a wiki-link when it has a vault path, else plain.
+
+    ``[[<vault-path-no-md>|<alias>]]`` for vault-mirrored docs (alias has its
+    ``[`` / ``]`` neutralized so Quartz's wiki-link regex matches); the bare
+    title otherwise. Honors the "never guess wiki-link targets" rule — the
+    target is always the doc's stored ``vault_path``, never a derived slug.
+    """
+    vault_path = vault_paths.get(doc_id)
+    if vault_path:
+        target = strip_md_extension(vault_path)
+        return f"[[{target}|{safe_wikilink_alias(title)}]]"
+    return title
+
+
+def _render_theme(block: ThemeBlock, vault_paths: dict[str, str]) -> list[str]:
+    """Render one theme block: heading, optional synthesis quote, doc bullets."""
+    heading = " & ".join(block.entity_names) if block.entity_names else block.key
+    lines = [f"### {heading}"]
+    if block.synthesis:
+        lines.append(f"> {block.synthesis}")
+    for doc_id, title in block.docs:
+        lines.append(f"- {_doc_link(doc_id, title, vault_paths)}")
+    return lines
+
+
+def _render_body(report: WeeklyReport) -> str:
+    """Build the Markdown body (everything after the frontmatter fence)."""
+    lines: list[str] = [
+        f"# Weekly review · {report.week}",
+        f"_{report.start_date.isoformat()} → {report.end_date.isoformat()}_",
+        "",
+        "## Themes this week",
+        "",
+    ]
+    if report.themes:
+        for i, block in enumerate(report.themes):
+            if i > 0:
+                lines.append("")
+            lines.extend(_render_theme(block, report.vault_paths))
+    else:
+        lines.append(_EMPTY_THEMES)
+
+    lines += ["", f"## Activity ({len(report.activity)} docs interacted with)", ""]
+    if report.activity:
+        lines.append("| Doc | Interactions |")
+        lines.append("|-----|-------------|")
+        for doc in report.activity:
+            link = _doc_link(doc.document_id, doc.title, report.vault_paths)
+            lines.append(f"| {link} | {doc.interaction_count} |")
+    else:
+        lines.append(_EMPTY_ACTIVITY)
+
+    lines += ["", "## Open loops", ""]
+    if report.open_loops:
+        for row in report.open_loops:
+            source = _doc_link(
+                row.document_id, row.document_title, report.vault_paths
+            )
+            lines.append(f"- [ ] {row.text}")
+            lines.append(f"  _from: {source}_")
+    else:
+        lines.append(_EMPTY_LOOPS)
+
+    lines += ["", f"## New captures ({len(report.ingested)} docs)", ""]
+    if report.ingested:
+        for ing in report.ingested:
+            link = _doc_link(ing.document_id, ing.title, report.vault_paths)
+            kind = ing.source_kind or "manual"
+            lines.append(f"- {link} ({kind})")
+    else:
+        lines.append(_EMPTY_INGESTED)
+
+    lines += ["", "## Key people", ""]
+    if report.key_people:
+        lines.append(" · ".join(report.key_people))
+    else:
+        lines.append(_EMPTY_PEOPLE)
+
+    return "\n".join(lines) + "\n"
+
+
+def render_weekly_md(report: WeeklyReport) -> str:
+    """Render the full vault page: YAML frontmatter + Markdown body.
+
+    Deterministic — two calls on the same report return byte-identical strings.
+    Frontmatter fields are emitted in a fixed order (``dump_frontmatter`` keeps
+    insertion order) so the page round-trips through
+    :func:`brain.vault.frontmatter.parse_frontmatter`.
+    """
+    fields = {
+        "kind": "review",
+        "week": report.week,
+        "date": report.generated_date.isoformat(),
+        "tags": ["review", "weekly"],
+        "window_start": report.start_date.isoformat(),
+        "window_end": report.end_date.isoformat(),
+    }
+    return dump_frontmatter(fields, _render_body(report))
+
+
+def render_weekly_json(report: WeeklyReport) -> dict[str, Any]:
+    """Render the machine-readable wire shape shared by CLI ``--json`` + MCP.
+
+    ``vault_path`` is the relative page slug (``reviews/<week>``) — present
+    whether or not the page was actually emitted, mirroring the MCP contract in
+    Plan 10 §3. Theme blocks expose ``community_key`` (the graph community key,
+    or the tag name on the fallback path) and ``doc_titles`` (not full doc refs).
+    """
+    return {
+        "week": report.week,
+        "start_date": report.start_date.isoformat(),
+        "end_date": report.end_date.isoformat(),
+        "vault_path": f"reviews/{report.week}",
+        "graph_used": report.graph_used,
+        "sections": {
+            "themes": [
+                {
+                    "community_key": block.key,
+                    "entity_names": block.entity_names,
+                    "doc_titles": [title for _id, title in block.docs],
+                    "synthesis": block.synthesis,
+                }
+                for block in report.themes
+            ],
+            "activity": [
+                {
+                    "document_id": doc.document_id,
+                    "title": doc.title,
+                    "interaction_count": doc.interaction_count,
+                }
+                for doc in report.activity
+            ],
+            "open_loops": [
+                {"document_title": row.document_title, "text": row.text}
+                for row in report.open_loops
+            ],
+            "ingested": [
+                {
+                    "document_id": ing.document_id,
+                    "title": ing.title,
+                    "source_kind": ing.source_kind,
+                }
+                for ing in report.ingested
+            ],
+            "key_people": report.key_people,
+        },
+    }
+
+
+def render_weekly_rich(report: WeeklyReport) -> str:
+    """Render a compact terminal summary (plain text, no Rich markup).
+
+    Used by ``brain review weekly --no-emit``. Leads with the report title so
+    callers / tests can assert on "Weekly review".
+    """
+    lines = [
+        f"Weekly review · {report.week}",
+        f"  {report.start_date.isoformat()} → {report.end_date.isoformat()}",
+        f"  themes: {len(report.themes)}  "
+        f"activity: {len(report.activity)}  "
+        f"open loops: {len(report.open_loops)}  "
+        f"captures: {len(report.ingested)}",
+    ]
+    for block in report.themes:
+        heading = " & ".join(block.entity_names) if block.entity_names else block.key
+        lines.append(f"  • {heading}")
+        if block.synthesis:
+            lines.append(f"    {block.synthesis}")
+    if report.key_people:
+        lines.append(f"  key people: {' · '.join(report.key_people)}")
+    return "\n".join(lines)