brainplug 0.4.0__py3-none-any.whl

brainplug/notion_api.py

@@ -0,0 +1,135 @@
+"""Read your Notion pages through the official Notion API.
+
+Why: a Notion *export* does not include the day a page was created (unless the
+page has a date property), so those pages come out as "no date". The API does
+return ``created_time`` for every page.
+
+Setup (once): create an internal integration at notion.so/my-integrations,
+share the pages you want with it (page menu > Connections), and put its secret
+in the ``NOTION_TOKEN`` environment variable. Then::
+
+    python -m brainplug build --notion -o my_brain.html
+
+Requests go from your computer straight to api.notion.com and nowhere else.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+import urllib.error
+import urllib.request
+from typing import Callable
+
+from .sources import MAX_NOTE_BYTES, Item, parse_date
+
+API = "https://api.notion.com/v1"
+VERSION = "2022-06-28"
+MAX_BLOCKS = 600  # per page
+MAX_DEPTH = 3
+
+
+class NotionError(ValueError):
+    pass
+
+
+def _call(token: str, method: str, url: str, body: dict | None = None, retries: int = 4) -> dict:
+    data = json.dumps(body).encode("utf-8") if body is not None else None
+    for attempt in range(retries + 1):
+        req = urllib.request.Request(url, data=data, method=method, headers={
+            "Authorization": f"Bearer {token}", "Notion-Version": VERSION,
+            "Content-Type": "application/json",
+        })
+        try:
+            with urllib.request.urlopen(req, timeout=30) as res:
+                return json.loads(res.read().decode("utf-8"))
+        except urllib.error.HTTPError as err:
+            if err.code == 429 and attempt < retries:  # rate limit: wait as long as Notion asks
+                time.sleep(min(30.0, float(err.headers.get("Retry-After") or 1)))
+                continue
+            if err.code == 401:
+                raise NotionError("Notion rejected the token. Check NOTION_TOKEN.") from err
+            if err.code in (403, 404):
+                raise NotionError("Notion: that page is not shared with your integration.") from err
+            raise NotionError(f"Notion API error {err.code}.") from err
+        except urllib.error.URLError as err:
+            raise NotionError(f"Could not reach Notion: {err.reason}") from err
+    raise NotionError("Notion API: too many retries.")
+
+
+def _plain(rich: list | None) -> str:
+    return "".join(str(part.get("plain_text", "")) for part in rich or [] if isinstance(part, dict))
+
+
+def _title(page: dict) -> str:
+    for prop in (page.get("properties") or {}).values():
+        if isinstance(prop, dict) and prop.get("type") == "title":
+            return _plain(prop.get("title")).strip()
+    return ""
+
+
+def _tags(page: dict) -> list[str]:
+    out = []
+    for prop in (page.get("properties") or {}).values():
+        if isinstance(prop, dict) and prop.get("type") == "multi_select":
+            out += [str(opt.get("name", "")) for opt in prop.get("multi_select") or []]
+    return sorted({t for t in out if t})
+
+
+def _blocks(token: str, base: str, block_id: str, depth: int, budget: list[int]) -> list[str]:
+    lines, cursor = [], None
+    while budget[0] > 0:
+        url = f"{base}/blocks/{block_id}/children?page_size=100" + (f"&start_cursor={cursor}" if cursor else "")
+        data = _call(token, "GET", url)
+        for block in data.get("results") or []:
+            budget[0] -= 1
+            kind = block.get("type")
+            if kind in ("child_page", "child_database"):
+                continue  # those are pages of their own
+            body = block.get(kind) if isinstance(block.get(kind), dict) else {}
+            text = _plain(body.get("rich_text"))
+            if text.strip() and kind != "code":
+                lines.append(text)
+            if block.get("has_children") and depth < MAX_DEPTH and budget[0] > 0:
+                lines += _blocks(token, base, block["id"], depth + 1, budget)
+        cursor = data.get("next_cursor")
+        if not data.get("has_more") or not cursor:
+            break
+    return lines
+
+
+def load_notion(token: str, base: str = API, say: Callable[[str], None] | None = None,
+                limit: int = 5000) -> list[Item]:
+    """Every page shared with the integration, with its real creation date."""
+    if not token:
+        raise NotionError("Set the NOTION_TOKEN environment variable to your integration secret.")
+    pages, cursor = [], None
+    while len(pages) < limit:
+        body = {"filter": {"property": "object", "value": "page"}, "page_size": 100}
+        if cursor:
+            body["start_cursor"] = cursor
+        data = _call(token, "POST", f"{base}/search", body)
+        pages += [p for p in data.get("results") or [] if not (p.get("archived") or p.get("in_trash"))]
+        cursor = data.get("next_cursor")
+        if not data.get("has_more") or not cursor:
+            break
+    items = []
+    for n, page in enumerate(pages[:limit], 1):
+        if say and n % 25 == 0:
+            say(f"Notion: {n}/{len(pages)} pages")
+        created = parse_date(page.get("created_time"))
+        if created is None:
+            continue
+        try:
+            lines = _blocks(token, base, page["id"], 0, [MAX_BLOCKS])
+        except NotionError:
+            continue  # one unreadable page should not stop the rest
+        text = re.sub(r"\s+", " ", re.sub(r"https?://\S+", " ", " ".join(lines))).strip()
+        if len(text.split()) < 3:
+            continue
+        items.append(Item(
+            id=f"notion:{page['id'].replace('-', '')}", title=_title(page) or "Untitled", created=created,
+            source="notion", kind="note", text=text[:MAX_NOTE_BYTES], tags=_tags(page),
+        ))
+    return items

brainplug/pipeline.py

@@ -0,0 +1,95 @@
+"""One function that goes from your files to the finished HTML.
+
+The command line (``build``), the sync command and the local app all call it,
+so they always behave the same.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Callable
+from zoneinfo import ZoneInfo
+
+from . import analysis, sources
+from .render import render
+
+
+def local_tz(name: str | None = None):
+    """A named time zone, or None for this computer's own zone. None (rather than
+    today's UTC offset) keeps summer and winter time right for every date."""
+    return ZoneInfo(name) if name else None
+
+
+@dataclass
+class Options:
+    inputs: list[str] = field(default_factory=list)
+    out: str = "brainplug.html"
+    me: list[str] = field(default_factory=list)  # your email(s) and your WhatsApp name
+    pdf: bool = False  # also read PDFs found inside folders
+    notion: bool = False  # read Notion through its API (NOTION_TOKEN)
+    sessions: bool = False  # also read Claude Code and Codex sessions on this computer
+    topics: int | None = None
+    private: bool = False
+    rename: dict[int, str] = field(default_factory=dict)
+    title: str = "My Brainplug"
+    lang: str = "en"
+    tz: str | None = None
+    vault: str | None = None  # write Markdown notes into this Obsidian vault
+    json: str | None = None  # also save the raw analysis as JSON
+    dataset: str | None = None  # also save a redacted JSON Lines dataset
+
+
+def describe(items: list[sources.Item]) -> str:
+    counts: dict[str, int] = {}
+    for item in items:
+        counts[item.source] = counts.get(item.source, 0) + 1
+    parts = [f"{n} from {analysis.SOURCE_NAMES.get(s, s.title())}"
+             for s, n in sorted(counts.items(), key=lambda kv: -kv[1])]
+    return f"Loaded {len(items)} items" + (": " + ", ".join(parts) if parts else "") + "."
+
+
+def load(o: Options, say: Callable[[str], None] = print) -> list[sources.Item]:
+    notes: list[str] = []
+    inputs = list(o.inputs) + ([str(p) for p in sources.session_folders()] if o.sessions else [])
+    items = sources.load(inputs, me=o.me, pdf=o.pdf, notes=notes)
+    if o.notion:
+        from .notion_api import load_notion
+        known = {i.id for i in items}
+        items += [i for i in load_notion(os.environ.get("NOTION_TOKEN", ""), say=say) if i.id not in known]
+        items.sort(key=lambda i: i.created)
+    for note in notes:
+        say(note)
+    return items
+
+
+def run(o: Options, say: Callable[[str], None] = print) -> tuple[list[sources.Item], dict]:
+    """Load, analyse and write everything that was asked for."""
+    items = load(o, say)
+    say(describe(items))
+    payload = analysis.analyse(items, tz=local_tz(o.tz), n_topics=o.topics, private=o.private,
+                               rename=o.rename, title=o.title, names=sources.topic_names_in(o.inputs))
+    say(f"Found {len(payload['topics'])} topics:")
+    for t in payload["topics"]:
+        say(f"  {t['id'] + 1:>2}. {t['name']:<34} {t['count']:>5} items  ({', '.join(t['terms'])})")
+    out = render(payload, o.out, lang=o.lang)
+    say(f"\nSaved {out.resolve()}")
+    if o.json:
+        Path(o.json).parent.mkdir(parents=True, exist_ok=True)
+        Path(o.json).write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
+        say(f"Saved {Path(o.json).resolve()}")
+    if o.vault:
+        from .export import export_vault
+        n = export_vault(items, payload, o.vault)
+        say(f"Wrote {n} notes to {Path(o.vault).resolve() / 'Brainplug'}")
+    if o.dataset:
+        from .export import export_dataset
+        n = export_dataset(items, payload, o.dataset)
+        say(f"Wrote {n} redacted items to {Path(o.dataset).resolve()}")
+        say("Redaction is best effort (emails, links, phone and long numbers). Read the file before sharing it.")
+    if o.private:
+        say("Private mode: titles and text are not in the HTML. Topic names still are,")
+        say("so check them (and use --rename) before sharing.")
+    return items, payload

brainplug/render.py

@@ -0,0 +1,47 @@
+"""Fill the HTML template with the analysis results.
+
+The output is a single self-contained file: the data, three.js and the font are
+all inlined, so it opens offline and nothing is ever sent anywhere.
+"""
+
+from __future__ import annotations
+
+import base64
+import html
+import json
+from pathlib import Path
+
+HERE = Path(__file__).parent
+TEMPLATE = HERE / "templates" / "viewer.html"
+THREE = HERE / "vendor" / "three.module.min.js"
+FONT = HERE / "vendor" / "inter-latin.woff2"
+
+
+def template_html() -> str:
+    """The viewer with three.js already inlined and the data still to fill in.
+    The web app (docs/app) ships this file and fills it in the browser."""
+    return (
+        TEMPLATE.read_text(encoding="utf-8")
+        .replace("__FONT__", base64.b64encode(FONT.read_bytes()).decode("ascii"))
+        .replace("__THREE__", THREE.read_text(encoding="utf-8"))
+    )
+
+
+def render_html(payload: dict, lang: str = "en") -> str:
+    data = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
+    # Your text may contain "</script>" or "<!--" (notes about code do). Written as \u003c,
+    # a "<" can never end or confuse the <script> block the data sits in.
+    data = data.replace("<", "\\u003c")
+    return (
+        template_html()
+        .replace("__LANG__", "es" if lang == "es" else "en")
+        .replace("__TITLE__", html.escape(payload["meta"]["title"]))
+        .replace("__DATA__", data)  # last, so your text is never scanned for placeholders
+    )
+
+
+def render(payload: dict, out: str | Path, lang: str = "en") -> Path:
+    out = Path(out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(render_html(payload, lang), encoding="utf-8")
+    return out