secondbrain-py 0.2.1__py3-none-any.whl

brain/ingest/pdf.py

@@ -0,0 +1,61 @@
+"""PDF extractor — pypdf primary, pdfplumber fallback."""
+from pathlib import Path
+
+from pypdf import PdfReader
+
+from . import ExtractedDoc
+
+
+def extract_pdf(path: Path) -> ExtractedDoc:
+    """Extract an :class:`ExtractedDoc` from a PDF file on disk.
+
+    Uses ``pypdf`` as the primary extractor. If no text is recovered (e.g. a
+    scanned PDF with images only), falls back to ``pdfplumber``, which handles
+    some layouts ``pypdf`` misses. Strips repeated header/footer lines that
+    appear on a majority of pages.
+    """
+    reader = PdfReader(str(path))
+    pages_text: list[str] = []
+    for page in reader.pages:
+        text = page.extract_text() or ""
+        pages_text.append(text.strip())
+
+    full_text = _strip_repeated_lines("\n\n".join(pages_text))
+
+    if not full_text.strip():  # pragma: no cover - only triggered by image-only PDFs
+        full_text = _fallback_pdfplumber(path)
+
+    return ExtractedDoc(
+        title=Path(path).stem,
+        content=full_text.strip(),
+        content_type="pdf",
+        source_path=str(Path(path).resolve()),
+        metadata={"page_count": len(reader.pages)},
+    )
+
+
+def _fallback_pdfplumber(path: Path) -> str:  # pragma: no cover - image-only PDFs
+    import pdfplumber
+
+    with pdfplumber.open(str(path)) as pdf:
+        return "\n\n".join((p.extract_text() or "").strip() for p in pdf.pages)
+
+
+def _strip_repeated_lines(text: str) -> str:
+    """Remove header/footer lines that appear on >50% of pages."""
+    pages = text.split("\n\n")
+    if len(pages) < 3:
+        return text
+    line_counts: dict[str, int] = {}
+    for page in pages:
+        for line in {ln.strip() for ln in page.splitlines() if ln.strip()}:
+            line_counts[line] = line_counts.get(line, 0) + 1
+    threshold = len(pages) // 2
+    repeated = {line for line, count in line_counts.items() if count > threshold}
+    if not repeated:
+        return text
+    cleaned_pages = [
+        "\n".join(ln for ln in page.splitlines() if ln.strip() not in repeated)
+        for page in pages
+    ]
+    return "\n\n".join(cleaned_pages)

brain/ingest/stdin.py

@@ -0,0 +1,22 @@
+"""Generic stdin ingester for Claude-orchestrated sources (Krisp, Slack, etc)."""
+from typing import Any
+
+from . import ExtractedDoc
+
+
+def make_doc(
+    *,
+    content: str,
+    title: str,
+    content_type: str,
+    source_path: str | None = None,
+    metadata: dict[str, Any] | None = None,
+) -> ExtractedDoc:
+    """Build an :class:`ExtractedDoc` from raw stdin content plus metadata."""
+    return ExtractedDoc(
+        title=title,
+        content=content.strip(),
+        content_type=content_type,
+        source_path=source_path,
+        metadata=metadata or {},
+    )

brain/ingest/sub_tokens.py

@@ -0,0 +1,91 @@
+"""Sub-token extractor — surface email/URL/hostname components for FTS."""
+import re
+
+# Standard noise TLDs / very common suffix words that add no retrieval value
+# when emitted as standalone sub-tokens. Kept lowercase; comparison is
+# case-insensitive.
+_NOISE_TOKENS: frozenset[str] = frozenset(
+    {"com", "org", "net", "io", "co", "gov", "edu"}
+)
+
+# Match emails: local@host.tld[.tld...]
+_EMAIL_RE = re.compile(r"[\w.+\-]+@[\w.\-]+\.\w+", re.UNICODE)
+
+# Match URLs (http/https). Stops at whitespace and a few common closing
+# punctuation chars so trailing `)`/`]`/`>` don't get folded into the URL.
+_URL_RE = re.compile(r"https?://[^\s)\]>]+", re.UNICODE)
+
+# Match bare hostnames: at least two dot-separated word groups, e.g.
+# ``example.com/groups``. Restricted to word chars + hyphen so it doesn't
+# match every dotted phrase.
+_HOST_RE = re.compile(r"\b[\w\-]+(?:\.[\w\-]+)+\b", re.UNICODE)
+
+# Splitter applied to each captured email/URL/host to break it into the
+# individual word components. Includes ``@`` so email captures split on
+# the local/host boundary, and ``:`` so any leftover scheme markers (e.g.
+# port suffixes) don't fold a port number into the host word.
+_SPLIT_RE = re.compile(r"[./+_@:?#&=]+", re.UNICODE)
+
+
+def _is_useful(token: str) -> bool:
+    """Filter predicate — drop tokens too short or too noisy to keep."""
+    if len(token) <= 1:
+        return False
+    if token.isdigit():
+        return False
+    return token.lower() not in _NOISE_TOKENS
+
+
+def _split_components(value: str) -> list[str]:
+    """Split a captured email/URL/hostname into its component words."""
+    return [part for part in _SPLIT_RE.split(value) if part]
+
+
+def extract_sub_tokens(text: str) -> str:
+    """Extract sub-tokens from emails, URLs, and bare hostnames in ``text``.
+
+    Returns a single whitespace-joined string of word components in
+    *first-seen order* (deduplicated). Returns ``""`` when ``text`` has no
+    matches or is empty. Pure function — safe to call twice; idempotent in
+    the sense that re-running it on its own output never crashes and
+    produces a sane subset (no new sub-tokens to find since the output is
+    already plain space-separated words with no `.`/`@`/`/`).
+
+    Filters: tokens of length ≤ 1, digits-only tokens, and the noise
+    suffixes in :data:`_NOISE_TOKENS` are dropped.
+    """
+    if not text:
+        return ""
+
+    seen: set[str] = set()
+    ordered: list[str] = []
+
+    def _add(token: str) -> None:
+        if not _is_useful(token):
+            return
+        key = token.lower()
+        if key in seen:
+            return
+        seen.add(key)
+        ordered.append(token)
+
+    for match in _EMAIL_RE.finditer(text):
+        for component in _split_components(match.group(0)):
+            _add(component)
+
+    for match in _URL_RE.finditer(text):
+        url = match.group(0)
+        # Strip the scheme so we don't emit "https" / "http" as a sub-token.
+        if "://" in url:
+            url = url.split("://", 1)[1]
+        for component in _split_components(url):
+            _add(component)
+
+    for match in _HOST_RE.finditer(text):
+        host = match.group(0)
+        # Skip bare numerics like "1.2.3" (each part already filtered as
+        # digits-only, but we'd still touch the dedup table). Cheap guard.
+        for component in _split_components(host):
+            _add(component)
+
+    return " ".join(ordered)

brain/ingest/text.py

@@ -0,0 +1,16 @@
+"""Plain-text extractor."""
+from pathlib import Path
+
+from . import ExtractedDoc
+
+
+def extract_text(path: Path) -> ExtractedDoc:
+    """Extract an :class:`ExtractedDoc` from a plain-text file on disk."""
+    content = Path(path).read_text(encoding="utf-8", errors="replace")
+    return ExtractedDoc(
+        title=Path(path).stem,
+        content=content.strip(),
+        content_type="txt",
+        source_path=str(Path(path).resolve()),
+        metadata={},
+    )

brain/interactions.py

@@ -0,0 +1,205 @@
+"""Append-only interaction log writes (one row per user event).
+
+Each row in ``interactions`` records one feedback event (open / rate /
+pin / click). Q1-C populated this table from two DOCUMENT surfaces:
+
+- CLI: ``brain rate <id> useful|irrelevant`` → source='cli', action one
+  of ``rated_useful`` / ``rated_irrelevant``, ``session_id=NULL``.
+- MCP: ``brain_show(..., originating_query=...)`` → source='mcp',
+  action='opened', ``query=originating_query``, optional
+  ``session_id`` minted by a prior ``brain_search`` call.
+
+The wiki click surface (source='wiki') is reserved in the schema for a
+future wave; this module accepts it today so the deferred surface needs
+no additional migration.
+
+G4-a (migration 015, spec §17d Q2) generalizes the writer so the graph
+surfaces — entity / community / theme — become FIRST-CLASS rateable
+targets. A row now targets EITHER a document (``document_id``) OR a graph
+target (``target_type`` + ``target_id``), never both and never neither —
+the XOR enforced both here (Python boundary) and by the authoritative DB
+``CHECK``. ``graph_retrieved`` is a PROVENANCE flag (a graph surface
+produced this row), independent of the target shape: a document row
+surfaced via a graph path is still a document row with
+``graph_retrieved=True``. No new ``source`` value is introduced.
+
+Per plan §3.a, the writer is intentionally narrow: callers go through
+:func:`record_interaction` instead of issuing INSERTs directly so the
+Python-side enum + XOR validation can produce a clean
+:class:`InteractionError` ahead of the authoritative DB-level ``CHECK``
+constraints.
+"""
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Literal
+
+import psycopg
+
+from .errors import InteractionError
+
+InteractionAction = Literal[
+    "clicked", "opened", "rated_useful", "rated_irrelevant", "pinned"
+]
+InteractionSource = Literal["cli", "mcp", "wiki"]
+InteractionTargetType = Literal["entity", "community", "theme"]
+
+
+# Python-side enum gates — kept in sync with the SQL ``CHECK`` constraints
+# on ``interactions.action`` / ``interactions.source`` (migration 010) and
+# ``interactions.target_type`` (migration 015).
+# Adding a new value here without the matching migration would silently
+# bypass the Python gate but still trip the SQL ``CHECK``; adding to the
+# SQL side without updating these sets would let a Python caller pass an
+# enum value the DB rejects at INSERT time. Keep both in lockstep.
+_VALID_ACTIONS: frozenset[str] = frozenset({
+    "clicked", "opened", "rated_useful", "rated_irrelevant", "pinned",
+})
+_VALID_SOURCES: frozenset[str] = frozenset({"cli", "mcp", "wiki"})
+_VALID_TARGET_TYPES: frozenset[str] = frozenset({"entity", "community", "theme"})
+
+
+@dataclass(frozen=True)
+class InteractionRow:
+    """Read-back projection of one ``interactions`` row.
+
+    Used by tests + a future ``brain interactions show`` command (not
+    exposed yet — the table is write-only from the user's perspective for
+    now). ``id`` and ``document_id`` are stringified UUIDs so callers
+    don't need to import ``uuid`` just to compare; ``document_id`` is
+    ``None`` for graph-target rows (G4-a). ``target_type`` / ``target_id``
+    are set (and ``document_id`` ``None``) for graph-target rows, mutually
+    exclusive with ``document_id`` per the XOR. ``graph_retrieved`` is the
+    provenance flag, orthogonal to the target shape.
+    """
+
+    id: str
+    document_id: str | None
+    query: str | None
+    action: str
+    source: str
+    session_id: str | None
+    at: datetime
+    target_type: str | None = None
+    target_id: str | None = None
+    graph_retrieved: bool = False
+
+
+def record_interaction(
+    conn: psycopg.Connection[Any],
+    *,
+    document_id: str | None = None,
+    action: InteractionAction,
+    source: InteractionSource,
+    query: str | None = None,
+    session_id: uuid.UUID | None = None,
+    target_type: InteractionTargetType | None = None,
+    target_id: str | None = None,
+    graph_retrieved: bool = False,
+) -> str:
+    """INSERT one row into ``interactions`` and return its UUID as text.
+
+    Validates ``action`` / ``source`` / ``target_type`` against the
+    Python-side enums and enforces the document-XOR-graph-target shape
+    before issuing the SQL, so an obvious mistake produces a clean
+    :class:`brain.errors.InteractionError` rather than a generic
+    :class:`psycopg.errors.CheckViolation`. The DB-level ``CHECK``
+    constraints remain the authoritative gate; this is belt-and-braces.
+
+    A row targets EITHER a document OR a graph target, never both and
+    never neither (migration 015 / spec §17d Q2):
+
+    - Document row: pass ``document_id``; leave ``target_type`` /
+      ``target_id`` unset. This is the unchanged Q1-C path used by
+      ``brain rate`` and MCP ``brain_show``.
+    - Graph-target row: pass BOTH ``target_type`` and ``target_id``;
+      leave ``document_id`` unset.
+
+    Args:
+        conn: Live Postgres connection. The caller controls the
+            transaction — for CLI / MCP we run with ``autocommit=True``
+            so each interaction is one round-trip.
+        document_id: Stringified UUID of the doc the event applies to.
+            Foreign-key enforced; deleting the doc cascades to its
+            interaction rows. ``None`` (the default) for graph-target rows.
+        action: One of :data:`_VALID_ACTIONS`. Type-narrowed by
+            :data:`InteractionAction` for static checkers.
+        source: One of :data:`_VALID_SOURCES`. Type-narrowed by
+            :data:`InteractionSource` for static checkers.
+        query: Optional originating query (the search string that led
+            the user to this doc). ``None`` is valid for surfaces with
+            no query intent (e.g., a direct CLI ``brain show``).
+        session_id: Optional UUID grouping a search-then-open pair.
+            ``None`` for the CLI rating path; populated by MCP when the
+            client passes back the id returned from ``brain_search``.
+        target_type: One of :data:`_VALID_TARGET_TYPES` ('entity' /
+            'community' / 'theme') for a graph-target row; ``None`` for a
+            document row. Must be paired with ``target_id``.
+        target_id: Durable id of the graph target (entity UUID /
+            community_key / theme key) as text; ``None`` for a document
+            row. Must be paired with ``target_type``.
+        graph_retrieved: Provenance flag — ``True`` when a graph surface
+            produced this interaction. Orthogonal to the target shape: a
+            document row surfaced via a graph path is still a document row
+            with ``graph_retrieved=True``.
+
+    Returns:
+        The inserted row's UUID as a string.
+
+    Raises:
+        InteractionError: ``action`` / ``source`` / ``target_type`` is not
+            a recognised enum value, or the document-XOR-graph-target shape
+            is violated (both set, neither set, or a half-specified graph
+            target).
+        psycopg.Error: The INSERT itself failed (FK violation, DB
+            outage, etc.) — propagated unchanged so the CLI / MCP outer
+            wrappers can surface it with their framework's error type.
+    """
+    if action not in _VALID_ACTIONS:
+        raise InteractionError(f"unknown action: {action!r}")
+    if source not in _VALID_SOURCES:
+        raise InteractionError(f"unknown source: {source!r}")
+    if target_type is not None and target_type not in _VALID_TARGET_TYPES:
+        raise InteractionError(f"unknown target_type: {target_type!r}")
+
+    has_document = document_id is not None
+    has_target = target_type is not None or target_id is not None
+    if has_document and has_target:
+        raise InteractionError(
+            "interaction must target EITHER a document or a graph target, "
+            "not both (document_id is mutually exclusive with "
+            "target_type/target_id)"
+        )
+    if not has_document and not has_target:
+        raise InteractionError(
+            "interaction must target either a document (document_id) or a "
+            "graph target (target_type + target_id)"
+        )
+    if has_target and (target_type is None or target_id is None):
+        raise InteractionError(
+            "graph-target interaction requires BOTH target_type and target_id"
+        )
+
+    row = conn.execute(
+        """
+        INSERT INTO interactions
+            (document_id, query, action, source, session_id,
+             target_type, target_id, graph_retrieved)
+        VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+        RETURNING id::text
+        """,
+        (
+            document_id,
+            query,
+            action,
+            source,
+            str(session_id) if session_id is not None else None,
+            target_type,
+            target_id,
+            graph_retrieved,
+        ),
+    ).fetchone()
+    assert row is not None  # RETURNING always yields one row
+    return str(row[0])