substack-saved-mcp 0.1.0__py3-none-any.whl

substack_saved_mcp/__init__.py

@@ -0,0 +1,3 @@
+"""Substack Saved Posts MCP & Sync Application."""
+
+__version__ = "0.1.0"

substack_saved_mcp/cli.py

@@ -0,0 +1,413 @@
+"""Command Line Interface (CLI) for Substack Saved Posts MCP & Sync tool."""
+
+import sys
+
+import click
+
+from substack_saved_mcp.content_utils import format_post_for_llm, html_to_llm_text
+from substack_saved_mcp.database import (
+    get_post,
+    get_status,
+    init_db,
+    soft_delete_post,
+    upsert_post,
+)
+from substack_saved_mcp.database import (
+    list_audiences as db_list_audiences,
+)
+from substack_saved_mcp.database import (
+    list_posts as db_list_posts,
+)
+from substack_saved_mcp.database import (
+    list_publications as db_list_publications,
+)
+from substack_saved_mcp.database import (
+    search_posts as db_search_posts,
+)
+from substack_saved_mcp.mcp_server import run_server
+from substack_saved_mcp.substack_client import (
+    AuthRequiredError,
+    SubstackSavedPostsClient,
+    perform_interactive_login,
+)
+from substack_saved_mcp.sync import sync_saved_posts as run_sync
+
+
+@click.group()
+@click.version_option(version="0.1.0")
+def cli() -> None:
+    """Substack Saved Posts MCP & Sync Engine."""
+    pass
+
+
+@cli.command()
+def init() -> None:
+    """Initialize local SQLite database schema and FTS5 search index."""
+    init_db()
+    status = get_status()
+    click.echo(f"Initialized database at: {status.database_path}")
+    click.echo(f"Active saved posts: {status.total_saved_posts}")
+
+
+@cli.command()
+def login() -> None:
+    """Launch interactive browser window to log in to your Substack account."""
+    try:
+        state_file = perform_interactive_login()
+        click.echo(f"Session saved to: {state_file}")
+    except Exception as e:
+        click.secho(f"Error during login: {e}", fg="red")
+        sys.exit(1)
+
+
+@cli.command()
+@click.option(
+    "--force", is_flag=True, help="Force full resync instead of incremental stop."
+)
+def sync(force: bool) -> None:
+    """Sync saved posts from Substack account into local SQLite cache."""
+    click.echo("Starting Substack saved posts sync...")
+    result = run_sync(force=force)
+
+    if result.status == "success":
+        msg = f"Sync complete! Fetched {result.fetched_count} posts, upserted {result.upserted_count} posts."
+        if result.reconciled_count:
+            msg += f" Unsaved {result.reconciled_count} post(s) no longer on Substack's saved list."
+        click.secho(msg, fg="green")
+    elif result.status == "auth_required":
+        click.secho(f"Authentication required: {result.error_message}", fg="yellow")
+    else:
+        click.secho(f"Sync failed: {result.error_message}", fg="red")
+
+
+@cli.command()
+def serve() -> None:
+    """Run FastMCP stdio server for desktop clients (Claude Desktop, Goose, etc.)."""
+    init_db()
+    run_server()
+
+
+@cli.command()
+@click.argument("url")
+def save(url: str) -> None:
+    """Save/bookmark a Substack post by URL."""
+    init_db()
+    click.echo(f"Saving post: {url}...")
+    client = SubstackSavedPostsClient()
+    try:
+        post, confirmation = client.save_post(url)
+        saved_db_post = upsert_post(post)
+        click.secho(
+            f"Successfully saved '{saved_db_post.title}' to local cache!", fg="green"
+        )
+        click.echo(f"Published at: {saved_db_post.published_at or 'N/A'}")
+        click.echo(f"Saved at: {saved_db_post.saved_at}")
+        if confirmation != "confirmed":
+            click.secho(
+                f"Warning: could not confirm the bookmark was saved on Substack's own page "
+                f"(status: {confirmation}). Cached locally regardless; a future 'sync --force' "
+                "will correct it if the remote save didn't actually happen.",
+                fg="yellow",
+            )
+    except AuthRequiredError as e:
+        click.secho(f"Authentication required: {e}", fg="yellow")
+    except Exception as e:
+        click.secho(f"Error saving post: {e}", fg="red")
+
+
+@cli.command()
+@click.argument("url_or_id")
+def unsave(url_or_id: str) -> None:
+    """Unsave/unbookmark a Substack post by URL or local ID."""
+    init_db()
+    post = get_post(url_or_id)
+    if not post:
+        click.secho(f"Post '{url_or_id}' not found in local cache.", fg="yellow")
+        return
+
+    click.echo(f"Unsaving post '{post.title}'...")
+    client = SubstackSavedPostsClient()
+    confirmation = "click_failed"
+    try:
+        post_id = int(post.substack_post_id) if post.substack_post_id else None
+        confirmation = client.unsave_post(post.url, post_id=post_id)
+    except Exception as e:
+        click.echo(f"Remote unsave notice: {e}")
+
+    updated = soft_delete_post(post.url)
+    if updated:
+        click.secho(
+            f"Successfully unsaved '{post.title}' from local cache.", fg="green"
+        )
+        if confirmation != "confirmed":
+            click.secho(
+                f"Warning: could not confirm the bookmark was removed on Substack's own page (status: {confirmation}).",
+                fg="yellow",
+            )
+
+
+@cli.command()
+@click.argument("query")
+@click.option("--publication", help="Filter by publication name.")
+@click.option(
+    "--audience",
+    help="Filter by audience tier (e.g. everyone, only_paid). See 'audiences' command for cached values.",
+)
+@click.option(
+    "--published-after",
+    help="Only posts published on/after this ISO-8601 date (e.g. 2026-01-01).",
+)
+@click.option(
+    "--published-before", help="Only posts published on/before this ISO-8601 date."
+)
+@click.option(
+    "--saved-after", help="Only posts bookmarked on/after this ISO-8601 date."
+)
+@click.option(
+    "--saved-before", help="Only posts bookmarked on/before this ISO-8601 date."
+)
+@click.option("--limit", default=10, help="Maximum search results.")
+def search(
+    query: str,
+    publication: str | None,
+    audience: str | None,
+    published_after: str | None,
+    published_before: str | None,
+    saved_after: str | None,
+    saved_before: str | None,
+    limit: int,
+) -> None:
+    """Perform full-text search across cached saved posts."""
+    init_db()
+    results = db_search_posts(
+        query=query,
+        publication=publication,
+        audience=audience,
+        published_after=published_after,
+        published_before=published_before,
+        saved_after=saved_after,
+        saved_before=saved_before,
+        limit=limit,
+    )
+    if not results:
+        click.echo(f"No saved posts matched query '{query}'.")
+        return
+
+    click.echo(f"Found {len(results)} matching post(s):\n")
+    for idx, p in enumerate(results, 1):
+        click.secho(f"{idx}. {p.title}", fg="cyan", bold=True)
+        click.echo(f"   Publication : {p.publication_name}")
+        click.echo(
+            f"   Published   : {p.published_at or 'N/A'} | Saved: {p.saved_at or 'N/A'}"
+        )
+        click.echo(f"   Audience    : {p.audience or 'N/A'}")
+        if p.reading_time_minutes or p.word_count:
+            click.echo(
+                f"   Reading time: {p.reading_time_minutes or '?'} min ({p.word_count or '?'} words)"
+            )
+        click.echo(f"   URL         : {p.url}")
+        if p.excerpt:
+            click.echo(f"   Excerpt     : {p.excerpt[:120]}...")
+        if p.image_url:
+            click.echo(f"   Image       : {p.image_url}")
+        click.echo("")
+
+
+@cli.command(name="list")
+@click.option("--limit", default=10, help="Number of posts to display.")
+@click.option("--offset", default=0, help="Pagination offset.")
+@click.option("--publication", help="Filter by publication name.")
+@click.option(
+    "--audience",
+    help="Filter by audience tier (e.g. everyone, only_paid). See 'audiences' command for cached values.",
+)
+@click.option(
+    "--sort-by", type=click.Choice(["saved_at", "published_at"]), default="saved_at"
+)
+def list_cmd(
+    limit: int, offset: int, publication: str | None, audience: str | None, sort_by: str
+) -> None:
+    """List saved posts ordered by saved date or publication date."""
+    init_db()
+    posts = db_list_posts(
+        limit=limit,
+        offset=offset,
+        publication=publication,
+        audience=audience,
+        sort_by=sort_by,
+    )
+    if not posts:
+        click.echo("No saved posts found.")
+        return
+
+    click.echo(f"Saved Posts ({len(posts)} displayed):\n")
+    for idx, p in enumerate(posts, offset + 1):
+        reading = (
+            f" | {p.reading_time_minutes} min ({p.word_count} words)"
+            if p.reading_time_minutes
+            else ""
+        )
+        click.secho(f"{idx}. {p.title}", fg="cyan")
+        click.echo(
+            f"   Pub: {p.publication_name} | Saved: {p.saved_at or 'N/A'} | Published: {p.published_at or 'N/A'} | Audience: {p.audience or 'N/A'}{reading}"
+        )
+        click.echo(f"   URL: {p.url}\n")
+
+
+@cli.command()
+def publications() -> None:
+    """List all publications present in the cache."""
+    init_db()
+    pubs = db_list_publications()
+    if not pubs:
+        click.echo("No publications in cache.")
+        return
+
+    click.echo(f"Cached Publications ({len(pubs)} total):\n")
+    for p in pubs:
+        click.echo(
+            f"- {p.publication_name} ({p.post_count} saved post{'s' if p.post_count != 1 else ''})"
+        )
+
+
+@cli.command()
+def audiences() -> None:
+    """List all audience tiers present in the cache (e.g. everyone, only_paid)."""
+    init_db()
+    tiers = db_list_audiences()
+    if not tiers:
+        click.echo("No posts in cache.")
+        return
+
+    click.echo(f"Cached Audience Tiers ({len(tiers)} total):\n")
+    for t in tiers:
+        click.echo(
+            f"- {t.audience or 'unknown'} ({t.post_count} saved post{'s' if t.post_count != 1 else ''})"
+        )
+
+
+@cli.command()
+def status() -> None:
+    """Show cache statistics and last sync run info."""
+    init_db()
+    st = get_status()
+    click.echo(f"Database Path       : {st.database_path}")
+    click.echo(f"Active Saved Posts  : {st.total_saved_posts}")
+    click.echo(f"Unsaved Posts       : {st.total_unsaved_posts}")
+    click.echo(f"Total Publications  : {st.total_publications}")
+    click.echo(f"Last Successful Sync: {st.last_successful_sync or 'Never'}")
+    click.echo(f"Last Sync Status    : {st.last_sync_status or 'N/A'}")
+
+
+@cli.command(name="get-content")
+@click.argument("url_or_id")
+@click.option(
+    "--no-cache",
+    is_flag=True,
+    help="Don't store the fetched content in the local cache.",
+)
+def get_content(url_or_id: str, no_cache: bool) -> None:
+    """Fetch a saved post's full content and print it formatted for an LLM.
+
+    Uses the cached content_text if a previous fetch already stored it;
+    otherwise fetches the post's page and caches the result unless --no-cache
+    is given.
+    """
+    init_db()
+    post = get_post(url_or_id)
+    if not post:
+        click.secho(f"Post '{url_or_id}' not found in local cache.", fg="yellow")
+        return
+
+    if post.content_text:
+        click.echo(
+            format_post_for_llm(
+                title=post.title,
+                publication_name=post.publication_name,
+                url=post.url,
+                body_text=post.content_text,
+                author_name=post.author_name,
+                published_at=post.published_at,
+            )
+        )
+        return
+
+    click.echo(f"Fetching full content for '{post.title}'...", err=True)
+    client = SubstackSavedPostsClient()
+    try:
+        result = client.fetch_post_content(post.url)
+    except AuthRequiredError as e:
+        click.secho(f"Authentication required: {e}", fg="yellow")
+        return
+    except Exception as e:
+        click.secho(f"Error fetching content: {e}", fg="red")
+        return
+
+    body_html = result.get("body_html")
+    if not body_html:
+        click.secho(
+            "Could not find this post's full content on its page (Substack may have "
+            "changed how it embeds it, or this post is paywalled beyond your account's "
+            "access). Run 'substack-saved-mcp inspect-network' while opening this "
+            f"post ({post.url}) in the browser so we can capture the real content "
+            "source, then this command can be updated.",
+            fg="yellow",
+        )
+        return
+
+    body_text = html_to_llm_text(body_html)
+    if not no_cache:
+        post.content_text = body_text
+        post = upsert_post(post)
+
+    click.echo(
+        format_post_for_llm(
+            title=post.title,
+            publication_name=post.publication_name,
+            url=post.url,
+            body_text=body_text,
+            author_name=post.author_name,
+            published_at=post.published_at,
+        )
+    )
+
+
+@cli.command()
+def inspect_network() -> None:
+    """Inspect and capture Substack saved posts network endpoints structure safely."""
+    click.echo("Launching Playwright inspector context...")
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError:
+        click.secho("Playwright not installed.", fg="red")
+        return
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=False)
+        context = browser.new_context()
+        page = context.new_page()
+
+        def handle_response(response):
+            if (
+                "api/v1" in response.url
+                or "bookmark" in response.url
+                or "saved" in response.url
+            ):
+                click.echo(
+                    f"[Network Intercept] {response.request.method} {response.url} (Status: {response.status})"
+                )
+                post_data = response.request.post_data
+                if post_data:
+                    click.echo(f"    Body: {post_data}")
+
+        page.on("response", handle_response)
+        page.goto("https://substack.com/saved")
+        click.echo(
+            "Navigate around your saved posts page. Press ENTER in terminal when finished."
+        )
+        input("--> Press ENTER to finish network inspection: ")
+        browser.close()
+
+
+if __name__ == "__main__":
+    cli()

substack_saved_mcp/config.py

@@ -0,0 +1,55 @@
+"""Configuration settings and filesystem path management."""
+
+import os
+from pathlib import Path
+
+APP_NAME = "substack-saved-mcp"
+
+
+def get_default_data_dir() -> Path:
+    """Return OS-appropriate application data directory."""
+    if env_dir := os.getenv("SUBSTACK_SAVED_DATA_DIR"):
+        return Path(env_dir).expanduser().resolve()
+
+    xdg_data = os.getenv("XDG_DATA_HOME")
+    if xdg_data:
+        return (Path(xdg_data) / APP_NAME).resolve()
+
+    # Default to user home local share or fallback
+    return (Path.home() / ".local" / "share" / APP_NAME).resolve()
+
+
+def get_db_path() -> Path:
+    """Return path to SQLite database."""
+    if env_db := os.getenv("SUBSTACK_SAVED_DB_PATH"):
+        return Path(env_db).expanduser().resolve()
+    return get_default_data_dir() / "saved_posts.sqlite"
+
+
+def get_browser_dir() -> Path:
+    """Return path to Playwright browser context / storage state directory."""
+    if env_browser := os.getenv("SUBSTACK_SAVED_BROWSER_DIR"):
+        return Path(env_browser).expanduser().resolve()
+    return get_default_data_dir() / "browser_state"
+
+
+def get_storage_state_path() -> Path:
+    """Return path to storage_state.json."""
+    return get_browser_dir() / "storage_state.json"
+
+
+def ensure_app_dirs() -> None:
+    """Ensure data and browser state directories exist with restrictive permissions (0o700)."""
+    data_dir = get_default_data_dir()
+    browser_dir = get_browser_dir()
+
+    data_dir.mkdir(parents=True, exist_ok=True)
+    browser_dir.mkdir(parents=True, exist_ok=True)
+
+    # Restrict permissions to user-only (read/write/execute) on POSIX systems
+    if os.name == "posix":
+        try:
+            data_dir.chmod(0o700)
+            browser_dir.chmod(0o700)
+        except Exception:
+            pass

substack_saved_mcp/content_utils.py

@@ -0,0 +1,151 @@
+"""Convert Substack post HTML content into clean text suitable for feeding to an LLM."""
+
+from html.parser import HTMLParser
+
+_BLOCK_TAGS = {
+    "p",
+    "div",
+    "section",
+    "article",
+    "blockquote",
+    "pre",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "ul",
+    "ol",
+    "li",
+    "tr",
+    "table",
+    "figure",
+    "figcaption",
+    "hr",
+}
+_SKIP_CONTENT_TAGS = {"script", "style", "noscript", "iframe"}
+_HEADING_PREFIX = {
+    "h1": "# ",
+    "h2": "## ",
+    "h3": "### ",
+    "h4": "#### ",
+    "h5": "##### ",
+    "h6": "###### ",
+}
+
+
+class _PostBodyToTextParser(HTMLParser):
+    """Minimal HTML-to-markdown-ish-text converter for Substack post body HTML."""
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self._out: list[str] = []
+        self._skip_depth = 0
+        self._tag_stack: list[str] = []
+        self._link_href_stack: list[str | None] = []
+        self._list_item_open = False
+
+    def _write(self, text: str) -> None:
+        if text:
+            self._out.append(text)
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag in _SKIP_CONTENT_TAGS:
+            self._skip_depth += 1
+            return
+        if self._skip_depth:
+            return
+
+        self._tag_stack.append(tag)
+
+        if tag == "br":
+            self._write("\n")
+        elif tag in _HEADING_PREFIX:
+            self._write("\n\n" + _HEADING_PREFIX[tag])
+        elif tag == "li":
+            self._write("\n- ")
+            self._list_item_open = True
+        elif tag in _BLOCK_TAGS:
+            self._write("\n\n")
+        elif tag in ("strong", "b"):
+            self._write("**")
+        elif tag in ("em", "i"):
+            self._write("*")
+        elif tag == "a":
+            href = dict(attrs).get("href")
+            self._link_href_stack.append(href)
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in _SKIP_CONTENT_TAGS:
+            if self._skip_depth:
+                self._skip_depth -= 1
+            return
+        if self._skip_depth:
+            return
+
+        if self._tag_stack and self._tag_stack[-1] == tag:
+            self._tag_stack.pop()
+
+        if tag in ("strong", "b"):
+            self._write("**")
+        elif tag in ("em", "i"):
+            self._write("*")
+        elif tag == "a" and self._link_href_stack:
+            href = self._link_href_stack.pop()
+            if href and not href.startswith("#"):
+                self._write(f" ({href})")
+        elif tag in _BLOCK_TAGS:
+            self._write("\n\n")
+
+    def handle_data(self, data: str) -> None:
+        if self._skip_depth:
+            return
+        self._write(data)
+
+    def get_text(self) -> str:
+        raw = "".join(self._out)
+        lines = [line.strip() for line in raw.splitlines()]
+        collapsed: list[str] = []
+        blank_run = 0
+        for line in lines:
+            if not line:
+                blank_run += 1
+                if blank_run <= 1:
+                    collapsed.append("")
+            else:
+                blank_run = 0
+                collapsed.append(line)
+        return "\n".join(collapsed).strip()
+
+
+def html_to_llm_text(html: str) -> str:
+    """Strip a Substack post's ``body_html`` down to clean text for LLM consumption.
+
+    Converts headings to markdown-style ``#`` prefixes, list items to ``- ``
+    bullets, links to ``text (url)``, and collapses excess whitespace, while
+    dropping script/style/iframe content entirely.
+    """
+    parser = _PostBodyToTextParser()
+    parser.feed(html or "")
+    parser.close()
+    return parser.get_text()
+
+
+def format_post_for_llm(
+    title: str,
+    publication_name: str,
+    url: str,
+    body_text: str,
+    author_name: str | None = None,
+    published_at: str | None = None,
+) -> str:
+    """Assemble a post's metadata and cleaned body text into one LLM-ready document."""
+    header_lines = [f"Title: {title}", f"Publication: {publication_name}"]
+    if author_name:
+        header_lines.append(f"Author: {author_name}")
+    if published_at:
+        header_lines.append(f"Published: {published_at}")
+    header_lines.append(f"URL: {url}")
+
+    return "\n".join(header_lines) + "\n\n" + body_text