net-sift 0.1.0__py3-none-any.whl

net_sift/__init__.py

@@ -0,0 +1,3 @@
+"""Net-Sift: coverage-first social and web deep-search as an MCP server."""
+
+__version__ = "0.1.0"

net_sift/access/__init__.py

@@ -0,0 +1,2 @@
+"""Access layer: walled login platforms reached through the user's logged-in
+Chromium browser via OpenCLI. Keyless sources live in net_sift.engine.sources."""

net_sift/access/opencli.py

@@ -0,0 +1,320 @@
+"""Walled-platform access through OpenCLI (https://github.com/jackwener/opencli).
+
+OpenCLI drives the user's real logged-in Chrome via a browser-bridge extension and
+a local daemon, so login platforms are reached as the human, with no cookie
+copying or private-API reverse engineering. Built-in adapters cover Twitter/X,
+Reddit, Instagram, Facebook, Bilibili, Xiaohongshu, Zhihu, and more.
+
+net-sift shells to ``opencli <site> <command> -f json`` directly and normalizes the
+structured output into engine records, so the raw platform payload never reaches
+the agent. Adapters are discovered at runtime via ``opencli list`` rather than
+hardcoded, so net-sift tracks OpenCLI's adapter set as it changes.
+
+Desktop-only: requires Node >= 20.18.1, the OpenCLI runtime, Chrome with the
+bridge extension, and an existing logged-in session. When any of that is missing,
+access degrades to a declared gap, never a crash.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import urllib.request
+from datetime import datetime, timezone
+
+from ..engine.core import Record, parse_iso, rec
+
+OPENCLI_BIN = os.environ.get("NET_SIFT_OPENCLI_BIN", "opencli")
+_DAEMON_STATUS_URL = "http://127.0.0.1:19825/status"  # loopback, side-effect-free
+
+# Login platforms we route through OpenCLI. Keyless platforms (hackernews, etc.)
+# stay in the engine even though OpenCLI also offers them.
+WALLED_SITES = (
+    "twitter",
+    "x",
+    "reddit",
+    "instagram",
+    "facebook",
+    "bilibili",
+    "xiaohongshu",
+    "zhihu",
+)
+# Commands that mean "search this platform for a query", most to least preferred.
+_SEARCH_COMMANDS = ("search", "query", "feed")
+
+
+class OpenCLIUnavailable(RuntimeError):
+    pass
+
+
+def binary() -> str | None:
+    return shutil.which(OPENCLI_BIN)
+
+
+def daemon_status(timeout: int = 2) -> dict | None:
+    """Read OpenCLI's loopback status without starting the CLI (which has side
+    effects). None when the daemon is not reachable."""
+    try:
+        req = urllib.request.Request(_DAEMON_STATUS_URL, headers={"X-OpenCLI": "1"})
+        with urllib.request.urlopen(req, timeout=timeout) as r:
+            return json.loads(r.read(64 * 1024))
+    except Exception:
+        return None
+
+
+def available() -> bool:
+    """True when the binary exists, the daemon answers, and a browser is connected."""
+    if not binary():
+        return False
+    st = daemon_status()
+    if not st:
+        return False
+    return bool(st.get("extension_connected") or st.get("connected") or st.get("ready"))
+
+
+def status() -> str:
+    """One-line, secret-free summary for the doctor."""
+    if not binary():
+        return "opencli: not installed (npm i -g @jackwener/opencli, or OpenCLIApp)"
+    st = daemon_status()
+    if not st:
+        return "opencli: installed, daemon not running (open Chrome with the OpenCLI extension)"
+    if available():
+        return "opencli: connected"
+    return "opencli: daemon up, browser/extension not connected (open Chrome, enable the extension)"
+
+
+def run(
+    site: str,
+    command: str,
+    *args: str,
+    fmt: str = "json",
+    limit: int | None = None,
+    timeout: int = 90,
+):
+    """Run ``opencli <site> <command> [args] -f json`` and return parsed output."""
+    exe = binary()
+    if not exe:
+        raise OpenCLIUnavailable("opencli binary not found")
+    cmd = [exe, site, command, *args, "-f", fmt]
+    if limit is not None:
+        cmd += ["--limit", str(limit)]
+    try:
+        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+    except (OSError, subprocess.SubprocessError) as e:
+        raise OpenCLIUnavailable(f"opencli invocation failed: {e}") from e
+    if proc.returncode != 0:
+        raise OpenCLIUnavailable(
+            f"opencli {site} {command} exit {proc.returncode}: {proc.stderr.strip()[:200]}"
+        )
+    out = proc.stdout.strip()
+    if not out:
+        return []
+    try:
+        return json.loads(out)
+    except ValueError as e:
+        raise OpenCLIUnavailable(f"opencli {site} {command}: non-JSON output") from e
+
+
+def discover() -> dict[str, list[str]]:
+    """Map site -> [commands] from ``opencli list``. Empty when unavailable."""
+    exe = binary()
+    if not exe:
+        return {}
+    try:
+        proc = subprocess.run(
+            [exe, "list", "-f", "json"], capture_output=True, text=True, timeout=30
+        )
+        if proc.returncode == 0 and proc.stdout.strip():
+            return _parse_list_json(proc.stdout)
+    except (OSError, subprocess.SubprocessError):
+        pass
+    # Fall back to the plain-text listing shape ("site command  - description").
+    try:
+        proc = subprocess.run([exe, "list"], capture_output=True, text=True, timeout=30)
+        return _parse_list_text(proc.stdout) if proc.returncode == 0 else {}
+    except (OSError, subprocess.SubprocessError):
+        return {}
+
+
+def _parse_list_json(text: str) -> dict[str, list[str]]:
+    try:
+        data = json.loads(text)
+    except ValueError:
+        return {}
+    out: dict[str, list[str]] = {}
+    items = data if isinstance(data, list) else data.get("commands", [])
+    for it in items:
+        if not isinstance(it, dict):
+            continue
+        site = it.get("site") or it.get("namespace") or it.get("group")
+        command = it.get("command") or it.get("name") or it.get("action")
+        if site and command:
+            out.setdefault(site, []).append(command)
+    return out
+
+
+def _parse_list_text(text: str) -> dict[str, list[str]]:
+    out: dict[str, list[str]] = {}
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        parts = line.split()
+        if len(parts) >= 2 and parts[0].isidentifier():
+            out.setdefault(parts[0], []).append(parts[1])
+    return out
+
+
+# --- normalization --------------------------------------------------------
+
+_ID_KEYS = ("id", "rest_id", "note_id", "aweme_id", "pk", "objectID", "url")
+_URL_KEYS = ("url", "link", "permalink", "share_url", "webpage_url", "href")
+_TEXT_KEYS = (
+    "text",
+    "full_text",
+    "content",
+    "caption",
+    "caption_text",
+    "body",
+    "desc",
+    "description",
+    "summary",
+    "title",
+)
+_CREATED_KEYS = (
+    "created_at",
+    "createdAt",
+    "created_utc",
+    "timestamp",
+    "taken_at",
+    "published",
+    "date",
+    "time",
+)
+_ENGAGEMENT = {
+    "likes": ("likes", "like_count", "favorite_count", "digg_count", "favoriteCount"),
+    "comments": ("comments", "comment_count", "reply_count", "replies", "commentCount"),
+    "reposts": ("reposts", "retweet_count", "share_count", "shares", "forwards"),
+    "views": ("views", "play_count", "view_count", "viewCount"),
+    "score": ("score", "points", "upvotes", "ups"),
+}
+
+
+def _first(d: dict, keys) -> object:
+    for k in keys:
+        v = d.get(k)
+        if v not in (None, "", [], {}):
+            return v
+    return None
+
+
+def _author(d: dict) -> str | None:
+    for k in ("author", "username", "screen_name", "handle", "uploader", "nickname", "name"):
+        v = d.get(k)
+        if isinstance(v, str) and v:
+            return v
+    for k in ("user", "owner", "account"):
+        u = d.get(k)
+        if isinstance(u, dict):
+            for kk in ("screen_name", "username", "name", "nickname", "unique_id"):
+                if u.get(kk):
+                    return u[kk]
+    return None
+
+
+def _created(d: dict) -> str | None:
+    v = _first(d, _CREATED_KEYS)
+    if v is None:
+        return None
+    if isinstance(v, (int, float)) and v > 1_000_000_000:
+        return datetime.fromtimestamp(v, timezone.utc).isoformat()
+    if isinstance(v, str):
+        return v
+    return None
+
+
+def _text(d: dict) -> str:
+    parts = []
+    for k in ("title",):
+        if isinstance(d.get(k), str) and d[k]:
+            parts.append(d[k])
+    body = _first(d, [k for k in _TEXT_KEYS if k != "title"])
+    if isinstance(body, dict):
+        body = body.get("text") or ""
+    if isinstance(body, str) and body and body not in parts:
+        parts.append(body)
+    return "\n".join(parts)
+
+
+def normalize(item: dict, site: str) -> Record | None:
+    if not isinstance(item, dict):
+        return None
+    ident = _first(item, _ID_KEYS)
+    text = _text(item)
+    if ident is None and not text:
+        return None
+    extra = {}
+    for field, keys in _ENGAGEMENT.items():
+        v = _first(item, keys)
+        if isinstance(v, (int, float)):
+            extra[field] = v
+    return rec(
+        site,
+        ident or abs(hash(text)) % 10**12,
+        _first(item, _URL_KEYS) or "",
+        _author(item),
+        text,
+        _created(item),
+        extra,
+    )
+
+
+def _items(payload) -> list[dict]:
+    """Find the list of result objects in whatever envelope the adapter returns."""
+    if isinstance(payload, list):
+        return [x for x in payload if isinstance(x, dict)]
+    if isinstance(payload, dict):
+        for k in ("results", "data", "items", "notes", "tweets", "posts", "list"):
+            v = payload.get(k)
+            if isinstance(v, list):
+                return [x for x in v if isinstance(x, dict)]
+    return []
+
+
+def make_source(site: str, command: str):
+    """Build an engine source that searches one walled platform via OpenCLI."""
+
+    def source(query, since, until, budget):
+        payload = run(site, command, query, limit=min(budget, 100))
+        out: list[Record] = []
+        for item in _items(payload)[:budget]:
+            r = normalize(item, site)
+            if not r:
+                continue
+            when = parse_iso(r.get("created_at"))
+            if when and since and when < since:
+                continue
+            if when and until and when > until:
+                continue
+            out.append(r)
+        return out, False
+
+    source.__name__ = f"opencli_{site}"
+    return source
+
+
+def walled_sources(sites=WALLED_SITES) -> dict[str, object]:
+    """Discover installed walled adapters and build a source per searchable site."""
+    if not available():
+        return {}
+    catalog = discover()
+    out: dict[str, object] = {}
+    for site in sites:
+        cmds = catalog.get(site) or []
+        command = next((c for c in _SEARCH_COMMANDS if c in cmds), None)
+        if command:
+            out[site] = make_source(site, command)
+    return out

net_sift/cli.py

@@ -0,0 +1,159 @@
+"""net-sift command line: serve the MCP, run the doctor, print status, and install
+into Claude Code and Codex.
+
+The install writers are small, idempotent, and take explicit paths so they can be
+tested against a temp home without touching a real config.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+from pathlib import Path
+
+from . import __version__
+from . import doctor as doctor_mod
+from . import search as search_mod
+from . import status as status_mod
+
+MCP_ENTRY = {"command": "net-sift", "args": ["serve"]}
+
+
+# --- install writers (pure, testable) -------------------------------------
+
+
+def merge_json_mcp(path: Path) -> str:
+    """Add the net-sift MCP server to a Claude-style ~/.claude.json."""
+    data = {}
+    if path.exists():
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except ValueError:
+            return f"skipped {path} (not valid JSON; add net-sift manually)"
+    servers = data.setdefault("mcpServers", {})
+    if servers.get("net-sift") == MCP_ENTRY:
+        return f"already present in {path}"
+    servers["net-sift"] = MCP_ENTRY
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2), encoding="utf-8")
+    return f"registered MCP in {path}"
+
+
+def ensure_statusline(path: Path) -> str:
+    """Set the Claude Code status bar to net-sift in settings.json."""
+    data = {}
+    if path.exists():
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except ValueError:
+            return f"skipped {path} (not valid JSON; add statusLine manually)"
+    line = {"type": "command", "command": "net-sift status --line", "padding": 0}
+    if data.get("statusLine") == line:
+        return f"status bar already set in {path}"
+    data["statusLine"] = line
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2), encoding="utf-8")
+    return f"status bar set in {path}"
+
+
+def ensure_codex(path: Path) -> str:
+    """Add [mcp_servers.net-sift] to a Codex config.toml without rewriting the file."""
+    block = '\n[mcp_servers.net-sift]\ncommand = "net-sift"\nargs = ["serve"]\n'
+    existing = path.read_text(encoding="utf-8") if path.exists() else ""
+    if "[mcp_servers.net-sift]" in existing:
+        return f"already present in {path}"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(existing + block, encoding="utf-8")
+    return f"registered MCP in {path}"
+
+
+# --- commands -------------------------------------------------------------
+
+
+def _cmd_install(args) -> int:
+    home = Path.home()
+    print("Installing net-sift for Claude Code and Codex...\n")
+    for msg in (
+        merge_json_mcp(home / ".claude.json"),
+        ensure_statusline(home / ".claude" / "settings.json"),
+        ensure_codex(home / ".codex" / "config.toml"),
+    ):
+        print("  " + msg)
+
+    print("\nDependency check:")
+    print("  node   :", doctor_mod._node_version())
+    print(
+        "  opencli:",
+        "found"
+        if shutil.which("opencli")
+        else "missing (npm i -g @jackwener/opencli, or install OpenCLIApp)",
+    )
+    print("\n" + doctor_mod.render())
+    print("\nRestart your client so it picks up the MCP server.")
+    return 0
+
+
+def _cmd_serve(args) -> int:
+    from .mcp_server import main as serve
+
+    serve()
+    return 0
+
+
+def _cmd_doctor(args) -> int:
+    print(doctor_mod.render(probe=args.probe))
+    return 0
+
+
+def _cmd_status(args) -> int:
+    if args.line:
+        print(status_mod.line())
+    else:
+        print(json.dumps(status_mod.snapshot(), indent=2))
+    return 0
+
+
+def _cmd_search(args) -> int:
+    platforms = args.platforms.split(",") if args.platforms else None
+    res = search_mod.deep_search(
+        args.query, platforms=platforms, since=args.since, until=args.until, max_budget=args.max
+    )
+    print(json.dumps(res, indent=2, ensure_ascii=False))
+    return 0
+
+
+def main(argv: list | None = None) -> int:
+    ap = argparse.ArgumentParser(
+        prog="net-sift", description="Coverage-first social and web deep-search (MCP)."
+    )
+    ap.add_argument("--version", action="version", version=f"net-sift {__version__}")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    sub.add_parser(
+        "install", help="register the MCP + status bar in Claude Code and Codex"
+    ).set_defaults(func=_cmd_install)
+    sub.add_parser("serve", help="run the MCP server (stdio)").set_defaults(func=_cmd_serve)
+
+    d = sub.add_parser("doctor", help="what can net-sift reach right now")
+    d.add_argument("--probe", action="store_true", help="also live-check keyless sources")
+    d.set_defaults(func=_cmd_doctor)
+
+    s = sub.add_parser("status", help="connectivity snapshot")
+    s.add_argument("--line", action="store_true", help="one-line form for a status bar")
+    s.set_defaults(func=_cmd_status)
+
+    q = sub.add_parser("search", help="run a search locally (prints the summary)")
+    q.add_argument("query")
+    q.add_argument("--platforms", help="comma-separated source names")
+    q.add_argument("--since", help="YYYY-MM-DD")
+    q.add_argument("--until", help="YYYY-MM-DD")
+    q.add_argument("--max", type=int, default=200, help="per-source budget")
+    q.set_defaults(func=_cmd_search)
+
+    args = ap.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

net_sift/config.py

@@ -0,0 +1,25 @@
+"""Runtime configuration. Everything is resolved from the environment so the tool
+stays portable; no OS keychain, no secrets on disk in this repo."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+#: Root for session storage and caches. Override with NET_SIFT_HOME.
+HOME = Path(os.environ.get("NET_SIFT_HOME", "~/.net-sift")).expanduser()
+SESSIONS_DIR = HOME / "sessions"
+
+#: Engine defaults.
+DEFAULT_BUDGET = 2000  # per-source, per-window record cap
+DEFAULT_NEAR_DUP = 0.85  # same-source near-duplicate collapse threshold
+DEFAULT_MIN_REL = 0.1  # relevance floor for ranking
+
+#: Optional environment keys the tool will use if present (never required).
+OPTIONAL_ENV = {
+    "GITHUB_TOKEN": "lifts the GitHub Search API rate limit",
+}
+
+
+def ensure_dirs() -> None:
+    SESSIONS_DIR.mkdir(parents=True, exist_ok=True)

net_sift/doctor.py

@@ -0,0 +1,100 @@
+"""Health check: what can net-sift reach right now, and what to do if nothing.
+
+Keyless sources are listed (optionally probed live). Walled platforms report
+through OpenCLI without spending or writing. The scaffolding mirrors the pattern
+in Agent Reach (MIT).
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+from datetime import datetime, timedelta, timezone
+
+from . import config
+from .access import opencli
+from .engine import sources
+
+
+def report(probe: bool = False) -> dict:
+    """Structured health report. probe=True live-checks keyless sources (slower)."""
+    walled = opencli.walled_sources()
+    data = {
+        "opencli": opencli.status(),
+        "node": _node_version(),
+        "walled_available": sorted(walled.keys()),
+        "walled_supported": list(opencli.WALLED_SITES),
+        "keyless_sources": list(sources.SOURCES.keys()),
+        "optional_env": {
+            k: ("set" if os.environ.get(k) else "unset") + f" ({desc})"
+            for k, desc in config.OPTIONAL_ENV.items()
+        },
+    }
+    if probe:
+        data["keyless_reachable"] = _probe_keyless()
+    data["guidance"] = _guidance(walled)
+    return data
+
+
+def _node_version() -> str:
+    exe = shutil.which("node")
+    if not exe:
+        return "not installed (OpenCLI needs Node >= 20.18.1)"
+    import subprocess
+
+    try:
+        v = subprocess.run(
+            [exe, "--version"], capture_output=True, text=True, timeout=5
+        ).stdout.strip()
+        return v or "installed"
+    except (OSError, subprocess.SubprocessError):
+        return "installed"
+
+
+def _probe_keyless() -> dict:
+    now = datetime.now(timezone.utc)
+    out = {}
+    for name, fn in sources.SOURCES.items():
+        if name == "telegram":  # needs channels; skip in a generic probe
+            out[name] = "skipped (needs --channels)"
+            continue
+        try:
+            recs, _ = fn("test", now - timedelta(days=30), now, 3)
+            out[name] = f"ok ({len(recs)})"
+        except Exception as e:
+            out[name] = f"unreachable: {type(e).__name__}"
+    return out
+
+
+def _guidance(walled: dict) -> str:
+    if walled:
+        return f"Walled platforms ready via OpenCLI: {', '.join(sorted(walled))}."
+    if not opencli.binary():
+        return (
+            "No walled platform reachable. Install OpenCLI (npm i -g @jackwener/opencli "
+            "or the OpenCLIApp), open a Chromium browser with the OpenCLI extension, and log "
+            "into the platforms you want (X, Reddit, Instagram, Facebook, Bilibili, Xiaohongshu)."
+        )
+    return (
+        "OpenCLI is installed but no browser session is connected. Open a Chromium browser "
+        "with the OpenCLI extension enabled and log into the platforms you want."
+    )
+
+
+def render(probe: bool = False) -> str:
+    d = report(probe=probe)
+    lines = ["net-sift doctor", "=" * 40]
+    lines.append(f"opencli : {d['opencli']}")
+    lines.append(f"node    : {d['node']}")
+    lines.append(f"walled  : {', '.join(d['walled_available']) or 'none connected'}")
+    lines.append(f"keyless : {', '.join(d['keyless_sources'])}")
+    lines.append("env     :")
+    for k, v in d["optional_env"].items():
+        lines.append(f"  {k}: {v}")
+    if "keyless_reachable" in d:
+        lines.append("keyless reachability:")
+        for k, v in d["keyless_reachable"].items():
+            lines.append(f"  {k:14} {v}")
+    lines.append("")
+    lines.append(d["guidance"])
+    return "\n".join(lines)

net_sift/engine/__init__.py

@@ -0,0 +1 @@
+"""Net-Sift engine: sources, ranking, dedup, coverage. Stdlib-only."""