substack-saved-mcp 0.1.0__py3-none-any.whl

substack_saved_mcp/sync.py

@@ -0,0 +1,267 @@
+"""Sync engine for retrieving Substack saved posts into the local SQLite cache."""
+
+import logging
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from substack_saved_mcp.database import (
+    finish_sync_run,
+    get_post,
+    init_db,
+    reconcile_unsaved_posts,
+    start_sync_run,
+    upsert_post,
+)
+from substack_saved_mcp.models import SavedPost, SyncRun
+from substack_saved_mcp.substack_client import (
+    AuthRequiredError,
+    SubstackSavedPostsClient,
+)
+from substack_saved_mcp.url_utils import canonicalize_url
+
+logger = logging.getLogger(__name__)
+
+# Rough average adult reading speed used to derive reading time from word count.
+WORDS_PER_MINUTE = 200
+
+
+def _first_positive_int(source: dict[str, Any], *keys: str) -> int | None:
+    """Return the first of ``keys`` whose value coerces to a positive int, else None."""
+    for key in keys:
+        val = source.get(key)
+        if val is None:
+            continue
+        try:
+            n = int(val)
+        except (TypeError, ValueError):
+            continue
+        if n > 0:
+            return n
+    return None
+
+
+def parse_remote_post(raw_data: dict[str, Any]) -> SavedPost:
+    """Parse raw Substack API JSON payload or DOM dict into a typed SavedPost object."""
+    # Check if raw_data is a normalized dictionary from DOM extraction. Use an explicit
+    # marker: reader-API post objects also carry a "canonical_url" key and must instead
+    # go through the full API mapping below (canonicalization, post_date, id, etc.).
+    if raw_data.get("_dom"):
+        return SavedPost(
+            url=raw_data["canonical_url"],
+            title=raw_data.get("title") or "Untitled Substack Post",
+            publication_name=raw_data.get("publication_name") or "Substack",
+            publication_url=raw_data.get("publication_url"),
+            author_name=raw_data.get("author_name"),
+            excerpt=raw_data.get("excerpt"),
+            published_at=raw_data.get("published_at"),
+            saved_at=raw_data.get("saved_at"),
+            audience=raw_data.get("audience"),
+            is_saved=1,
+        )
+
+    # Substack API json mapping
+    post_obj = raw_data.get("post") or raw_data.get("item") or raw_data
+    pub_obj = raw_data.get("publication") or post_obj.get("publication") or {}
+
+    substack_id = str(post_obj.get("id")) if post_obj.get("id") else None
+    raw_url = post_obj.get("canonical_url") or post_obj.get("url") or ""
+    clean_url = canonicalize_url(raw_url)
+
+    title = post_obj.get("title") or "Untitled Substack Post"
+    pub_name = pub_obj.get("name") or "Substack"
+    pub_url = (
+        pub_obj.get("custom_domain")
+        or f"https://{pub_obj.get('subdomain')}.substack.com"
+        if pub_obj.get("subdomain")
+        else None
+    )
+    author = (
+        post_obj.get("author")
+        or post_obj.get("author_name")
+        or (pub_obj.get("author_name") if pub_obj else None)
+    )
+
+    # Dates. Leave saved_at unknown (None) rather than stamping the sync moment.
+    published_at = post_obj.get("post_date") or post_obj.get("published_at")
+    saved_at = raw_data.get("created_at") or raw_data.get("saved_at")
+
+    excerpt = (
+        post_obj.get("description")
+        or post_obj.get("subtitle")
+        or post_obj.get("excerpt")
+    )
+    content_text = post_obj.get("body_html") or post_obj.get("content_text")
+    image_url = post_obj.get("cover_image") or post_obj.get("image_url")
+    audience = post_obj.get("audience")
+    is_paywalled = 1 if audience == "only_paid" or post_obj.get("is_paywalled") else 0
+
+    # Word count: Substack's exact reader-API field name isn't confirmed, so map the
+    # most likely candidates defensively (verify with `inspect-network` if these stay
+    # empty). Reading time is *derived* from word count rather than read from a field,
+    # since a wrong guess about that field's unit (seconds vs minutes) would store a
+    # badly wrong value — deriving at ~WPM is unambiguous and is how Substack itself
+    # presents it.
+    word_count = _first_positive_int(post_obj, "wordcount", "word_count", "words")
+    reading_time_minutes = (
+        -(-word_count // WORDS_PER_MINUTE)
+        if word_count
+        else None  # ceil division, min 1
+    )
+
+    return SavedPost(
+        substack_post_id=substack_id,
+        url=clean_url,
+        title=title,
+        publication_name=pub_name,
+        publication_url=pub_url,
+        author_name=author,
+        published_at=published_at,
+        saved_at=saved_at,
+        is_saved=1,
+        excerpt=excerpt,
+        content_text=content_text,
+        image_url=image_url,
+        audience=audience,
+        is_paywalled=is_paywalled,
+        word_count=word_count,
+        reading_time_minutes=reading_time_minutes,
+    )
+
+
+def sync_saved_posts(
+    force: bool = False,
+    db_path: Path | None = None,
+    client: SubstackSavedPostsClient | None = None,
+) -> SyncRun:
+    """Execute an incremental or full sync of Substack saved posts into SQLite cache."""
+    init_db(db_path)
+    sync_mode = "full" if force else "incremental"
+    sync_id = start_sync_run(sync_mode=sync_mode, db_path=db_path)
+
+    active_client = client or SubstackSavedPostsClient()
+    active_client.reset_cache()
+    total_fetched = 0
+    total_upserted = 0
+    consecutive_matches = 0
+    MAX_CONSECUTIVE_MATCHES = 3
+    # Only populated meaningfully for a force/full sync, which enumerates every
+    # currently-saved remote post; an incremental sync may stop early and its
+    # partial list must never be used to infer removals.
+    remote_urls: list[str] = []
+
+    page_size = 50
+    offset = 0
+
+    try:
+        while True:
+            remote_items = active_client.fetch_saved_posts_page(
+                limit=page_size, offset=offset
+            )
+            if not remote_items:
+                break
+
+            total_fetched += len(remote_items)
+
+            for item in remote_items:
+                parsed_post = parse_remote_post(item)
+                if not parsed_post.url:
+                    continue
+                remote_urls.append(parsed_post.url)
+
+                # Incremental sync check
+                if not force:
+                    existing = get_post(parsed_post.url, db_path=db_path)
+                    if (
+                        existing
+                        and existing.is_saved == 1
+                        and existing.saved_at == parsed_post.saved_at
+                    ):
+                        consecutive_matches += 1
+                        if consecutive_matches >= MAX_CONSECUTIVE_MATCHES:
+                            logger.info(
+                                f"Incremental sync: encountered {consecutive_matches} existing matches. Stopping early."
+                            )
+                            break
+                    else:
+                        consecutive_matches = 0
+
+                upsert_post(parsed_post, db_path=db_path)
+                total_upserted += 1
+
+            if not force and consecutive_matches >= MAX_CONSECUTIVE_MATCHES:
+                break
+
+            if len(remote_items) < page_size:
+                break
+
+            offset += page_size
+
+        reconciled_count = 0
+        if force:
+            reconciled_count = reconcile_unsaved_posts(remote_urls, db_path=db_path)
+            if reconciled_count:
+                logger.info(
+                    f"Reconciliation: soft-deleted {reconciled_count} post(s) no longer in the remote saved list."
+                )
+
+        finish_sync_run(
+            sync_id=sync_id,
+            status="success",
+            fetched_count=total_fetched,
+            upserted_count=total_upserted,
+            reconciled_count=reconciled_count,
+            db_path=db_path,
+        )
+        return SyncRun(
+            id=sync_id,
+            started_at=datetime.now(UTC).isoformat(),
+            completed_at=datetime.now(UTC).isoformat(),
+            status="success",
+            sync_mode=sync_mode,
+            fetched_count=total_fetched,
+            upserted_count=total_upserted,
+            reconciled_count=reconciled_count,
+        )
+
+    except AuthRequiredError as e:
+        msg = str(e)
+        finish_sync_run(
+            sync_id=sync_id,
+            status="auth_required",
+            fetched_count=total_fetched,
+            upserted_count=total_upserted,
+            error_message=msg,
+            db_path=db_path,
+        )
+        return SyncRun(
+            id=sync_id,
+            started_at=datetime.now(UTC).isoformat(),
+            completed_at=datetime.now(UTC).isoformat(),
+            status="auth_required",
+            sync_mode=sync_mode,
+            fetched_count=total_fetched,
+            upserted_count=total_upserted,
+            error_message=msg,
+        )
+    except Exception as e:
+        msg = f"Sync failed: {e!s}"
+        logger.exception(msg)
+        finish_sync_run(
+            sync_id=sync_id,
+            status="failed",
+            fetched_count=total_fetched,
+            upserted_count=total_upserted,
+            error_message=msg,
+            db_path=db_path,
+        )
+        return SyncRun(
+            id=sync_id,
+            started_at=datetime.now(UTC).isoformat(),
+            completed_at=datetime.now(UTC).isoformat(),
+            status="failed",
+            sync_mode=sync_mode,
+            fetched_count=total_fetched,
+            upserted_count=total_upserted,
+            error_message=msg,
+        )

substack_saved_mcp/url_utils.py

@@ -0,0 +1,55 @@
+"""URL normalization and tracking parameter stripping utilities."""
+
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+
+# Common tracking parameters present in Substack links
+TRACKING_PARAMS = {
+    "utm_source",
+    "utm_medium",
+    "utm_campaign",
+    "utm_term",
+    "utm_content",
+    "r",
+    "s",
+    "ref",
+    "post_id",
+    "isFreemail",
+    "triedSigningIn",
+    "action",
+    "showWelcome",
+}
+
+
+def canonicalize_url(url: str) -> str:
+    """Normalize post URL by stripping tracking query params and trailing slash.
+
+    Example:
+      Input:  https://example.substack.com/p/my-post-slug?utm_source=substack&r=1a2b3#comments
+      Output: https://example.substack.com/p/my-post-slug
+    """
+    if not url:
+        return ""
+
+    parsed = urlparse(url.strip())
+    # Keep query parameters that are NOT in TRACKING_PARAMS
+    query_params = parse_qsl(parsed.query, keep_blank_values=False)
+    filtered_params = [(k, v) for k, v in query_params if k not in TRACKING_PARAMS]
+    clean_query = urlencode(filtered_params)
+
+    # Normalize path (remove trailing slash except for root path)
+    path = parsed.path
+    if len(path) > 1 and path.endswith("/"):
+        path = path[:-1]
+
+    # Reconstruct clean canonical URL (strip fragment/anchor as well)
+    clean_url = urlunparse(
+        (
+            parsed.scheme.lower(),
+            parsed.netloc.lower(),
+            path,
+            parsed.params,
+            clean_query,
+            "",  # fragment stripped
+        )
+    )
+    return clean_url

substack_saved_mcp-0.1.0.dist-info/METADATA

@@ -0,0 +1,218 @@
+Metadata-Version: 2.4
+Name: substack-saved-mcp
+Version: 0.1.0
+Summary: Local stdio-based MCP server and sync engine for Substack saved posts
+Author-email: Toni Hermoso Pulido <toniher@cau.cat>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: click>=8.1.0
+Requires-Dist: fastmcp>=0.1.0
+Requires-Dist: playwright>=1.40.0
+Requires-Dist: pydantic>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Requires-Dist: ruff==0.16.1; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Substack Saved Posts MCP & CLI
+
+A local, stdio-based Model Context Protocol (MCP) server and sync engine for your saved/bookmarked Substack posts.
+
+## Features
+
+- **Read & Search**: Full-text search (SQLite FTS5) across saved post titles, excerpts, authors, and publications. Filter by publication, audience tier (e.g. `everyone`, `only_paid`), and date ranges (`published_at` vs `saved_at`). Search also covers a post's **full body text**, but only for posts whose content has already been fetched once via `get-content` / the `get_post_content` tool — a normal `sync` stores metadata and excerpts, not full bodies, so posts you haven't opened yet are matched on their title/excerpt/metadata only, not their full text.
+- **Full Content for LLMs**: Fetch a saved post's full content and get it back cleaned and formatted (headings, lists, links) for feeding directly to an LLM, with the result cached locally for next time.
+- **Save & Unsave**: Bookmark new Substack posts or unbookmark existing ones via authenticated browser sessions.
+- **Offline First**: Fast, offline queries directly from local SQLite cache.
+- **Privacy & Security**: Keeps session credentials local, redacting tokens from logs.
+- **FastMCP Protocol**: Stdio MCP interface with rich tool suite and resources.
+
+---
+
+## Installation with `uv`
+
+[`uv`](https://github.com/astral-sh/uv) is the recommended fast Python package manager for installing and running `substack-saved-mcp`.
+
+### Option A: Install System-wide as a Tool (`uv tool install`)
+
+Install directly from your local repository folder:
+
+```bash
+# Navigate to the repository
+cd /path/to/substack-saved-mcp
+
+# Install system-wide into an isolated uv environment
+uv tool install .
+
+# Or install directly from a remote Git repository:
+# uv tool install git+https://github.com/your-username/substack-saved-mcp.git
+```
+
+After installation, `substack-saved-mcp` is immediately available in your PATH:
+
+```bash
+# Verify installation
+substack-saved-mcp --help
+```
+
+To update or uninstall:
+```bash
+# Upgrade installed tool
+uv tool upgrade substack-saved-mcp
+
+# Uninstall tool
+uv tool uninstall substack-saved-mcp
+```
+
+---
+
+### Option B: Local Development / Development Environment (`uv sync`)
+
+If you are developing or modifying the codebase:
+
+```bash
+# Clone and enter directory
+cd substack-saved-mcp
+
+# Install dependencies and dev tools (pytest)
+uv sync --extra dev
+
+# Run CLI commands using uv run
+uv run substack-saved-mcp --help
+
+# Run tests
+uv run pytest
+```
+
+---
+
+## Quick Start
+
+```bash
+# 1. Initialize local database
+substack-saved-mcp init
+
+# 2. Authenticate with Substack (opens interactive browser window once)
+substack-saved-mcp login
+
+# 3. Sync saved posts into local cache
+substack-saved-mcp sync
+
+# 4. Search saved posts via CLI
+substack-saved-mcp search "artificial intelligence"
+
+# 4b. Filter by publication or audience tier (see which tiers are cached with `audiences`)
+substack-saved-mcp audiences
+substack-saved-mcp list --audience only_paid
+substack-saved-mcp search "artificial intelligence" --audience everyone
+
+# 5. Save or unsave a post
+substack-saved-mcp save "https://example.substack.com/p/post-slug"
+substack-saved-mcp unsave "https://example.substack.com/p/post-slug"
+
+# 6. Get a saved post's full content, cleaned up and ready for an LLM
+substack-saved-mcp get-content "https://example.substack.com/p/post-slug"
+
+# 7. Launch stdio MCP server
+substack-saved-mcp serve
+```
+
+---
+
+## Configuring MCP Clients (Claude Desktop, Goose, Cursor, etc.)
+
+Add `substack-saved-mcp` to your MCP client's configuration file (e.g. `claude_desktop_config.json`).
+
+### Using System-Wide Installed Tool (`uv tool` or global binary)
+
+```json
+{
+  "mcpServers": {
+    "substack-saved": {
+      "command": "substack-saved-mcp",
+      "args": ["serve"]
+    }
+  }
+}
+```
+
+### Using `uv` directly from the Repository Path
+
+If you prefer running directly from your repository path without installing system-wide:
+
+```json
+{
+  "mcpServers": {
+    "substack-saved": {
+      "command": "uv",
+      "args": [
+        "--directory",
+        "/path/to/substack-saved-mcp",
+        "run",
+        "substack-saved-mcp",
+        "serve"
+      ]
+    }
+  }
+}
+```
+
+---
+
+## Frequently Asked Questions (FAQ)
+
+### Where is the database saved?
+
+By default, the SQLite database is saved in your OS application data directory:
+
+- **Linux / macOS**: `~/.local/share/substack-saved-mcp/saved_posts.sqlite`  
+  *(or `$XDG_DATA_HOME/substack-saved-mcp/saved_posts.sqlite` if `XDG_DATA_HOME` is set)*
+
+You can specify a custom database path or directory using environment variables:
+```bash
+export SUBSTACK_SAVED_DB_PATH="/path/to/my/custom_database.sqlite"
+# or
+export SUBSTACK_SAVED_DATA_DIR="/path/to/my/data_dir"
+```
+
+### Will a browser window pop up when running as an MCP server?
+
+**No, a visible browser window will not open during normal MCP operations.**
+
+- **Read & Search Tools** (`search_saved_posts`, `list_saved_posts`, `get_saved_post`, `list_publications`, `saved_posts_status`):  
+  Operate 100% offline using the local SQLite database. Zero browser activity.
+- **Sync & Write Tools** (`sync_saved_posts`, `save_post`, `unsave_post`, `get_post_content`):  
+  Run in **headless background mode** using the pre-authenticated session stored in `storage_state.json`.
+- **Interactive Login**:  
+  A visible browser window opens **only** when you manually run `substack-saved-mcp login` from your terminal. If your session expires while using an MCP client, the tool will return a clear error message instructing you to re-authenticate via `substack-saved-mcp login` instead of popping open a browser window unexpectedly.
+
+### What if I get a Playwright "Executable doesn't exist" error?
+
+If you encounter an error like `BrowserType.launch: Executable doesn't exist` when running commands (especially `login`), it means Playwright hasn't installed its required browsers in the isolated environment.
+
+To fix this, you need to run the `playwright install` command *inside* the environment where the tool is installed. 
+
+For a system-wide tool installation (via `uv tool install`), run:
+```bash
+~/.local/share/uv/tools/substack-saved-mcp/bin/playwright install
+```
+
+If you are using a local development environment (via `uv sync`), run:
+```bash
+uv run playwright install
+```
+
+### I edited the source code, but the installed `substack-saved-mcp` command still behaves like the old version. Why?
+
+`uv tool install` copies the package into its own isolated environment at install time — it does **not** track your working tree. If you edited files under `src/` (or pulled new commits) after installing the tool system-wide, the globally installed copy is stale and keeps running the old code, even though `uv run substack-saved-mcp ...` from the repo would use the latest source.
+
+Reinstall from your current working tree to pick up the changes:
+```bash
+uv tool install . --no-cache --force
+```
+- `--force` replaces the existing installed version instead of skipping the install because a version is already present.
+- `--no-cache` ensures a fresh build rather than reusing a cached wheel/build artifact from before your edits.
+
+Do this any time after modifying the codebase and before relying on the globally installed `substack-saved-mcp` binary (as opposed to `uv run substack-saved-mcp`, which always reflects the working tree).

substack_saved_mcp-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+substack_saved_mcp/__init__.py,sha256=Eqt8L6DOMTXLExCNmSqhBIkZJqvg32gDf_dF2y65mYA,74
+substack_saved_mcp/cli.py,sha256=GbabUl9dGO48-2d6lQLc7CCHWgMguAAs-7Uz-QHF6Ms,13755
+substack_saved_mcp/config.py,sha256=oDZveru_tPZigMvEyYkvKsJbNhupj51btgWI_zk4sqI,1757
+substack_saved_mcp/content_utils.py,sha256=pFADyAmXhtAO-fMdxMWO_ynf30Ez-1UKpu-g-O2gEqQ,4231
+substack_saved_mcp/database.py,sha256=gmW0kQ-ehZn3z7BhF3DiLYy0AnN3dLMfF42N1Bak1k4,21562
+substack_saved_mcp/mcp_server.py,sha256=rDs793FT1yIwvjg_CDkeYvKBtL1NWVS-5sEK3R7VGpQ,10413
+substack_saved_mcp/models.py,sha256=g01TWK8N0EBuzuZEhJ2WrSe9I8QkeQk8t_yGjBaw8Ro,2738
+substack_saved_mcp/substack_client.py,sha256=Gsh_XJ9zCCrA3ilZdUE0jbMGLxghFh5tEuQ_jhLmRfU,28873
+substack_saved_mcp/sync.py,sha256=04TVwBS8vP0DWoIgoZ4ERLGnVEsuJLu505nyYmHS3qw,9433
+substack_saved_mcp/url_utils.py,sha256=WIswnksq_VLKvcHl-K7cAaE71L4M0LASpkBRxHunL6Y,1530
+substack_saved_mcp-0.1.0.dist-info/METADATA,sha256=38RMyKCkehej-VMzM6zUfVLNjIumVoNnwtDAThBEWFM,7937
+substack_saved_mcp-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+substack_saved_mcp-0.1.0.dist-info/entry_points.txt,sha256=3gZ2dPi4HRivOBMPaO3TcaW2NUR978nydg7LtcY3O0s,66
+substack_saved_mcp-0.1.0.dist-info/licenses/LICENSE,sha256=ZTJDgcyOdyNUW1neSxNwjSrNz0mZ7EYm3tXr_FbXdTU,1076
+substack_saved_mcp-0.1.0.dist-info/RECORD,,

substack_saved_mcp-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

substack_saved_mcp-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+substack-saved-mcp = substack_saved_mcp.cli:cli

substack_saved_mcp-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Toni Hermoso Pulido
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.