sharepoint-knowledge 0.1.0__py3-none-any.whl

sharepoint_knowledge/__init__.py

@@ -0,0 +1,3 @@
+"""sharepoint-knowledge — SharePoint connector on the shared engine."""
+
+__version__ = "0.1.0"

sharepoint_knowledge/auth_sp.py

@@ -0,0 +1,118 @@
+"""Entra auth for sharepoint-knowledge — device-code flow + silent refresh.
+
+The MSAL token cache is stored as a credential blob through the shared
+keyring/file machinery (same posture as the Confluence PAT). The access
+token itself never enters chat, logs, or command-line arguments.
+
+SP_KNOWLEDGE_TOKEN env var short-circuits everything (tests/CI).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+from confluence_knowledge.auth import get_credential, store_credential
+from confluence_knowledge.settings import SetupError
+
+SCOPES = ["https://graph.microsoft.com/Sites.Read.All", "offline_access"]
+ENV_TOKEN = "SP_KNOWLEDGE_TOKEN"
+AUTH_FILE = ".sp_auth.json"
+INDEX_DIR = ".sp_knowledge_index"
+SERVICE = "sharepoint-knowledge"
+
+
+def _cache_ref(profile) -> str:
+    return f"cache:{profile.credential_ref}"
+
+
+def _load_msal():
+    try:
+        import msal
+    except ImportError as e:
+        raise SetupError(
+            "msal isn't installed — reinstall with `pip install "
+            "sharepoint-knowledge[models]`") from e
+    return msal
+
+
+def _load_cache(workdir: Path, profile):
+    msal = _load_msal()
+    blob = get_credential(workdir, _cache_ref(profile), env=ENV_TOKEN,
+                          service=SERVICE, auth_file=AUTH_FILE,
+                          home_dir=".sharepoint-knowledge")
+    if blob:
+        try:
+            cache = msal.SerializableTokenCache()
+            cache.deserialize(json.loads(blob))
+            return cache
+        except Exception:  # noqa: BLE001 — corrupt cache → re-sign-in
+            return None
+    return None
+
+
+def _save_cache(workdir: Path, home_dir: Path, profile, cache) -> str:
+    return store_credential(workdir, home_dir, _cache_ref(profile),
+                            json.dumps(cache.serialize()),
+                            service=SERVICE, auth_file=AUTH_FILE,
+                            index_dir=INDEX_DIR)
+
+
+def _app(profile, cache=None):
+    msal = _load_msal()
+    kwargs = {"token_cache": cache} if cache else {}
+    return msal.PublicClientApplication(
+        profile.client_id,
+        authority=f"{_get_auth_base()}/{profile.tenant_id}",
+        **kwargs)
+
+
+def _get_auth_base() -> str:
+    """Authority base override for e2e (fake token endpoint)."""
+    return os.environ.get("SP_KNOWLEDGE_AUTH_BASE",
+                          "https://login.microsoftonline.com")
+
+
+def acquire_token(profile, workdir: Path, home_dir: Path) -> str:
+    """Silent first (cached refresh token); device flow only on expiry —
+    so normal use needs no re-sign-in after setup."""
+    if os.environ.get(ENV_TOKEN):
+        return os.environ[ENV_TOKEN]
+    cache = _load_cache(workdir, profile)
+    app = _app(profile, cache)
+    accounts = app.get_accounts()
+    if accounts:
+        r = app.acquire_token_silent(SCOPES, account=accounts[0])
+        if r and "access_token" in r:
+            if cache and cache.has_state_changed:
+                _save_cache(workdir, home_dir, profile, cache)
+            return r["access_token"]
+    raise SetupError(
+        "SharePoint sign-in expired — run `sharepoint-knowledge setup` to "
+        "re-authenticate (device code, one browser step)")
+
+
+def device_sign_in(profile, workdir: Path, home_dir: Path) -> dict:
+    """Interactive device-code sign-in — used by `setup`. Prints the
+    message Microsoft returns (URL + code); blocks until approved or it
+    times out. Returns the stored-cache location description."""
+    msal = _load_msal()
+    cache = msal.SerializableTokenCache()
+    app = _app(profile, cache)
+    flow = app.initiate_device_flow(scopes=SCOPES)
+    if "user_code" not in flow:
+        raise SetupError(
+            "couldn't start device sign-in: "
+            f"{flow.get('error_description', flow)}")
+    print("\n" + flow["message"] + "\n")  # "go to microsoft.com/devicelogin…"
+    r = app.acquire_token_by_device_flow(flow)
+    if "access_token" not in r:
+        raise SetupError(
+            "sign-in failed: "
+            f"{r.get('error_description', r.get('error', r))}")
+    where = _save_cache(workdir, home_dir, profile, cache)
+    account = r.get("id_token_claims", {})
+    return {"where": where,
+            "user": (account.get("preferred_username")
+                     or account.get("email") or "signed in")}

sharepoint_knowledge/cli_sp.py

@@ -0,0 +1,191 @@
+"""sharepoint-knowledge CLI — setup (Entra device sign-in), doctor,
+prepare-models, register, update, serve, clean.
+
+Same shape as the other knowledge CLIs. `setup` prints the exact Entra
+app-registration steps for IT, collects tenant/client/site, then runs the
+one-time device-code sign-in.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from confluence_knowledge.registration import register as _register
+from confluence_knowledge.retrieval import prepare_models
+from confluence_knowledge.settings import SetupError, ensure_gitignore
+
+from .settings_sp import (
+    APP_NAME,
+    AUTH_ENV,
+    AUTH_FILE_NAME,
+    INDEX_DIR_NAME,
+    PROFILE_NAME,
+    SPProfile,
+    home,
+    workdir,
+)
+
+TOKEN_STEPS = """\
+Entra app registration — once per tenant, done by IT (or you, if allowed):
+  1. Entra portal (entra.microsoft.com) → Applications → App registrations
+     → New registration
+  2. Name: e.g. "sharepoint-knowledge-agent" — Accounts in this org only
+  3. Authentication → Enable "Allow public client flows" → YES (required
+     for device-code sign-in — no client secret is used or needed)
+  4. API permissions → Add → Microsoft Graph → Delegated → Sites.Read.All
+     (+ offline_access is implied by device flow)
+  5. Grant admin consent for the tenant (the delegated scope normally
+     requires it) — one click on the Permissions page
+  6. Copy: Directory (tenant) ID and Application (client) ID — needed below
+
+This app requests only delegated read access — it can read exactly what
+the signed-in user can already read, nothing more.
+"""
+
+
+def _setup() -> int:
+    print("SharePoint Knowledge — first-run setup\n")
+    print(TOKEN_STEPS)
+
+    tenant = input("Tenant ID (GUID): ").strip()
+    client = input("Client ID (application ID, GUID): ").strip()
+    site = input("SharePoint site URL (e.g. https://contoso.sharepoint.com"
+                 "/sites/team): ").strip().rstrip("/")
+    if not (tenant and client and site):
+        print("tenant ID, client ID, and site URL are all required",
+              file=sys.stderr)
+        return 1
+
+    profile = SPProfile(tenant_id=tenant, client_id=client,
+                        site_url=site, credential_ref="sp-token")
+    profile.save()
+    print(f"\nprofile saved to {home() / PROFILE_NAME}\n")
+
+    print("Sign-in (device code — one browser step). The code you get "
+          "never leaves the OS credential store.", flush=True)
+    if os.environ.get(AUTH_ENV):
+        print("SP_KNOWLEDGE_TOKEN is set — using it; skipping sign-in")
+    else:
+        from .auth_sp import device_sign_in
+        try:
+            r = device_sign_in(profile, workdir(), home())
+            print(f"signed in as {r['user']} — token cache stored at "
+                  f"{r['where']}")
+        except SetupError as e:
+            print(f"sign-in failed: {e}", file=sys.stderr)
+            return 1
+
+    # make sure the workdir's gitignore hides the index + auth file
+    workdir().mkdir(parents=True, exist_ok=True)
+    ensure_gitignore(workdir(), [AUTH_FILE_NAME,
+                                 f"{INDEX_DIR_NAME}/"])
+
+    if input("\nDownload the two local models now (~175 MB, one-time)? "
+             "[Y/n] ").strip().lower() not in ("n", "no"):
+        for line in prepare_models(home()):
+            print(line)
+
+    print("\nsetup complete — run `sharepoint-knowledge doctor` to verify")
+    return 0
+
+
+def _doctor() -> int:
+    from confluence_knowledge.update import check
+
+    from .server_sp import status
+    s = status()
+    print(json.dumps(s, indent=2))
+    upd = check("sharepoint-knowledge")
+    if upd["installed"]:
+        line = f"version: {upd['installed']}"
+        if upd["latest"]:
+            line += f" (latest: {upd['latest']})"
+        elif upd["latest"] is None:
+            line += " (latest: unchecked — offline or internal mirror)"
+        print(line)
+    if upd["update_available"]:
+        print("update available: run `sharepoint-knowledge update` "
+              "(or pip install -U sharepoint-knowledge[models])")
+    return 0
+
+
+def main() -> None:
+    p = argparse.ArgumentParser(prog="sharepoint-knowledge",
+                                description="Local SharePoint indexer + "
+                                            "grounded-Q&A MCP service")
+    sub = p.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("setup", help="configure Entra sign-in + site")
+    sub.add_parser("doctor", help="report configuration + model readiness")
+    sub.add_parser("prepare-models", help="download the pinned local models")
+    sub.add_parser("update", help="upgrade this package via pip -U")
+    reg = sub.add_parser("register",
+                         help="write the IDE MCP config automatically")
+    reg.add_argument("--ide", choices=["auto", "all", "jetbrains", "vscode"],
+                     default="auto")
+    reg.add_argument("--workspace", default=None,
+                     help="repo dir for the workspace .vscode/mcp.json")
+    reg.add_argument("--remove", action="store_true",
+                     help="remove our server entry instead of adding it")
+    sub.add_parser("serve", help="run the stdio MCP server")
+    sub.add_parser("clean", help="delete all indexed data")
+
+    args = p.parse_args()
+    if args.cmd == "setup":
+        sys.exit(_setup())
+    if args.cmd == "doctor":
+        sys.exit(_doctor())
+    if args.cmd == "prepare-models":
+        for line in prepare_models(home()):
+            print(line)
+        return
+    if args.cmd == "update":
+        import subprocess
+        cmd = [sys.executable, "-m", "pip", "install", "-U",
+               "sharepoint-knowledge[models]"]
+        if os.name == "nt":
+            print("on Windows, upgrade via the venv python directly:\n  "
+                  + " ".join(f'"{c}"' if " " in c else c for c in cmd))
+            return
+        r = subprocess.run(cmd, check=False)
+        if r.returncode == 0:
+            print("\nupdated — models are pinned and unchanged; "
+                  "run `sharepoint-knowledge doctor` to confirm")
+        sys.exit(r.returncode)
+    if args.cmd == "register":
+        results = _register(
+            ide=args.ide,
+            workspace=Path(args.workspace) if args.workspace else None,
+            remove=args.remove,
+            server_id=APP_NAME,
+            module="sharepoint_knowledge.server_sp",
+            env_var="SP_KNOWLEDGE_WORKDIR")
+        for path, action in results:
+            print(f"{action}: {path}")
+        if args.remove:
+            print("removed — reload the IDE to drop the server")
+        else:
+            print("registered — reload the IDE so the tools appear")
+        return
+    if args.cmd == "serve":
+        from .server_sp import main as serve
+        serve()
+        return
+    if args.cmd == "clean":
+        import shutil
+
+        from .settings_sp import index_root
+        d = index_root()
+        if d.exists():
+            shutil.rmtree(d)
+            print(f"deleted {d}")
+        else:
+            print("nothing to clean")
+        return
+
+
+if __name__ == "__main__":
+    main()

sharepoint_knowledge/client_sp.py

@@ -0,0 +1,69 @@
+"""GET-only Microsoft Graph client for sharepoint-knowledge.
+
+Reuses the allowlist + GET-only machinery of the Confluence client: every
+request passes through the same `_validate_path` — the only Graph routes
+allowed are the read ones the crawler needs. File downloads use the
+pre-authenticated `@microsoft.graph.downloadUrl` fetched WITHOUT the Bearer
+header — the token never leaves graph.microsoft.com.
+"""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import urlparse
+
+import httpx
+from confluence_knowledge.client import (
+    AccessError,
+    ConfluenceClient,
+    EndpointRule,
+    SecurityException,
+)
+
+GRAPH_RULES = [
+    EndpointRule(re.compile(r"sites/[^/]+:.*"), "site by hostname:path"),
+    EndpointRule(re.compile(r"sites/[^/]+$"), "site info"),
+    EndpointRule(re.compile(r"sites/[^/]+/lists/?$"), "site lists"),
+    EndpointRule(re.compile(r"sites/[^/]+/lists/[^/]+/items/?$"), "list items"),
+    EndpointRule(re.compile(r"sites/[^/]+/lists/[^/]+/items/[^/]+/?$"), "list item"),
+    EndpointRule(re.compile(r"sites/[^/]+/pages/?$"), "site pages"),
+    EndpointRule(re.compile(r"sites/[^/]+/pages/[^/]+/?$"), "site page"),
+    EndpointRule(re.compile(r"sites/[^/]+/drives/?$"), "document libraries"),
+    EndpointRule(re.compile(r"sites/[^/]+/drive/?$"), "default drive"),
+    EndpointRule(re.compile(r"sites/[^/]+/drive/root:.+"), "item by path"),
+    EndpointRule(re.compile(r"drives/[^/]+/root/children/?$"), "library root"),
+    EndpointRule(re.compile(r"drives/[^/]+/items/[^/]+/children/?$"), "folder children"),
+    EndpointRule(re.compile(r"drives/[^/]+/items/[^/]+/?$"), "item metadata"),
+    EndpointRule(re.compile(r"me/?$"), "identity"),
+]
+
+
+def make_client(base_url: str, token: str) -> ConfluenceClient:
+    """Graph client — Bearer stays on the API host only."""
+    return ConfluenceClient(base_url, {"Authorization": f"Bearer {token}"},
+                            GRAPH_RULES)
+
+
+def download_file(url: str, max_bytes: int = 25 * 1024 * 1024) -> bytes:
+    """Fetch a pre-authenticated downloadUrl — NO Authorization header.
+
+    The URL embeds a short-lived download token; sending our Bearer would
+    leak it to a different host (and is unnecessary — these links are
+    self-authenticating). https only (localhost allowed for tests).
+    """
+    u = urlparse(url)
+    if u.scheme != "https" and u.hostname not in ("localhost", "127.0.0.1"):
+        raise SecurityException(
+            "download URL isn't https — refusing to fetch")
+    with httpx.Client(timeout=60.0, follow_redirects=True) as http, \
+            http.stream("GET", url) as r:
+        # bounded stream — don't buffer an unbounded response
+        if r.status_code >= 400:
+            raise AccessError(f"download failed: HTTP {r.status_code}")
+        out = bytearray()
+        for chunk in r.iter_bytes(1 << 20):
+            out += chunk
+            if len(out) > max_bytes:
+                raise AccessError(
+                    f"file exceeds {max_bytes} bytes — refusing to buffer")
+        return bytes(out)

sharepoint_knowledge/crawl_sp.py

@@ -0,0 +1,277 @@
+"""Durable BFS over a SharePoint site (pages + document libraries).
+
+Queue entries: site:<id> → list:<list_id> + drive:<drive_id>;
+list:<id> → page:<item_id>…; drive:<id> → item:<drive_id>:<item_id>…
+File bytes are routed through the shared extract_files (docx/pdf/pptx/xlsx/
+csv/txt/md/html). Site-page canvas HTML goes through extract_chunks.
+Only items reachable through the chosen root are ever fetched.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from urllib.parse import urlparse
+
+from confluence_knowledge.client import AccessError, ConfluenceClient
+from confluence_knowledge.extract import extract_chunks
+from confluence_knowledge.extract_files import extract_file_chunks, is_secret_name
+from confluence_knowledge.index import Index
+from confluence_knowledge.settings import SetupError
+
+MAX_FILE_BYTES = 25 * 1024 * 1024
+MAX_FILES = 20_000
+
+
+def resolve_root(client: ConfluenceClient, url: str) -> tuple[str, str]:
+    """SharePoint URL → (root queue entry, site webUrl for citations).
+
+    Accepts:
+      https://host.sharepoint.com/sites/name  → whole site (pages+libraries)
+      …/sites/name/<library or folder path>   → that subtree only
+      https://host.sharepoint.com             → root site
+    """
+    u = urlparse(url.strip())
+    if u.scheme != "https" and u.hostname not in ("localhost", "127.0.0.1"):
+        raise SetupError("SharePoint URL must be https")
+    host = u.hostname
+    path = u.path.rstrip("/") or "/"
+
+    # site path = /sites/{name} or /teams/{name} (or root "/")
+    site_path, item_path = "/", ""
+    parts = [p for p in path.split("/") if p]
+    if parts and parts[0] in ("sites", "teams"):
+        site_path = "/" + "/".join(parts[:2])
+        item_path = "/".join(parts[2:])
+        site_path = site_path.rstrip("/")
+
+    if site_path == "/":
+        site = client.get(f"sites/{host}").json()
+    else:
+        site = client.get(f"sites/{host}:{site_path}").json()
+    site_id = site["id"]
+    site_web = site.get("webUrl", url)
+
+    if item_path:
+        item = client.get(
+            f"sites/{site_id}/drive/root:/{item_path}").json()
+        drive = (item.get("parentReference") or {}).get("driveId") or \
+            client.get(f"sites/{site_id}/drive").json()["id"]
+        return f"item:{drive}:{item['id']}", site_web
+    return f"site:{site_id}", site_web
+
+
+def _paged(client: ConfluenceClient, path: str, params: dict | None = None):
+    """Follow @odata.nextLink — absolute links are validated inside the
+    client's origin+base by _validate_path, so pagination stays in scope."""
+    while True:
+        r = client.get(path, params)
+        d = r.json()
+        yield from d.get("value", [])
+        nxt = d.get("@odata.nextLink")
+        if not nxt:
+            return
+        path, params = nxt, None
+
+
+class SPCrawler:
+    def __init__(self, client: ConfluenceClient, index: Index, embedder=None,
+                 downloader=None):
+        self.client = client
+        self.index = index
+        self.embedder = embedder
+        self.downloader = downloader  # injected for tests; real one in server
+
+    # ---- expansions ----
+
+    def _expand_site(self, site_id: str) -> tuple[list[str], list[str]]:
+        """site → the Site Pages list + every document library."""
+        out, warns = [], []
+        for lst in _paged(self.client, f"sites/{site_id}/lists"):
+            name = lst.get("displayName", "")
+            base = (lst.get("list") or {}).get("template", "")
+            if name == "Site Pages" or base == "webPageLibrary":
+                out.append(f"list:{site_id}:{lst['id']}")
+        for drv in _paged(self.client, f"sites/{site_id}/drives"):
+            out.append(f"drive:{drv['id']}")
+        return out, warns
+
+    def _expand_drive(self, drive_id: str) -> list[str]:
+        return [f"item:{drive_id}:{it['id']}"
+                for it in _paged(self.client, f"drives/{drive_id}/root/children")]
+
+    def _expand_item(self, drive_id: str, item_id: str) -> list[str]:
+        return [f"item:{drive_id}:{it['id']}"
+                for it in _paged(self.client,
+                                 f"drives/{drive_id}/items/{item_id}/children")]
+
+    # ---- pages ----
+
+    def _page_from_list_item(self, site_id: str, list_id: str,
+                             item_id: str) -> tuple[dict, list, list]:
+        d = self.client.get(
+            f"sites/{site_id}/lists/{list_id}/items/{item_id}",
+            {"$expand": "fields"}).json()
+        fields = d.get("fields", {})
+        title = fields.get("Title") or d.get("name") or item_id
+        canvas = fields.get("CanvasContent1") or fields.get("WikiField") or ""
+        warns: list[str] = []
+        chunks = []
+        if canvas:
+            html = canvas
+            try:
+                parts = json.loads(canvas)
+                if isinstance(parts, list):
+                    html = "\n".join(p.get("innerHtml", "")
+                                     for p in parts if isinstance(p, dict))
+            except (json.JSONDecodeError, TypeError):
+                pass  # raw HTML already
+            chunks, warns = extract_chunks(html)
+        else:
+            warns.append("no canvas content — page may be empty or classic")
+        page = {"page_id": f"page:{item_id}", "title": title,
+                "url": (d.get("webUrl")
+                        or (fields.get("_dlc_DocIdUrl") or {}).get("Url", "")),
+                "version": str(d.get("lastModifiedDateTime") or ""),
+                "content_hash": hashlib.sha256(canvas.encode()).hexdigest(),
+                "warnings": warns}
+        return page, chunks, warns
+
+    def _page_from_item(self, drive_id: str, item_id: str,
+                        downloader) -> tuple[dict, list, list]:
+        d = self.client.get(
+            f"drives/{drive_id}/items/{item_id}",
+            {"$select": "id,name,size,webUrl,lastModifiedDateTime,"
+                        "@microsoft.graph.downloadUrl"}).json()
+        name = d.get("name", item_id)
+        warns: list[str] = []
+        if is_secret_name(name):
+            # disclose-and-skip — same posture as local files: the file is
+            # never a page in the index, only a warning on the job
+            return None, [], [f"{name}: looks like a secret — skipped"]
+        dl_url = d.get("@microsoft.graph.downloadUrl")
+        if not dl_url or (d.get("size") or 0) > MAX_FILE_BYTES:
+            why = ("no download URL" if not dl_url
+                   else f"too large ({d['size']} bytes)")
+            return None, [], [f"{name}: {why} — skipped"]
+        data = downloader(dl_url)
+        chunks, warns = extract_file_chunks(name, data)
+        return {"page_id": f"file:{item_id}", "title": name,
+                "url": d.get("webUrl", ""),
+                "version": str(d.get("lastModifiedDateTime", "")),
+                "content_hash": hashlib.sha256(data).hexdigest(),
+                "warnings": warns}, chunks, warns
+
+    # ---- the crawl ----
+
+    def start(self, root_entry: str) -> str:
+        job_id = uuid.uuid4().hex[:12]
+        self.index.abandon_running_jobs()
+        self.index.create_job(job_id, root_entry)
+        self.index.save_job(job_id, "running", [root_entry], [])
+        return job_id
+
+    def _checkpoint(self, job_id, queue, errors, seen, levels, warns) -> None:
+        with self.index.lock, self.index.db:
+            self.index.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)",
+                                  (f"seen:{job_id}", json.dumps(sorted(seen))))
+            self.index.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)",
+                                  (f"levels:{job_id}", json.dumps(levels)))
+            self.index.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)",
+                                  (f"warns:{job_id}", json.dumps(warns)))
+            self.index.db.execute(
+                "UPDATE jobs SET queue=?, errors=? WHERE job_id=?",
+                (json.dumps(queue), json.dumps(errors), job_id))
+
+    def step(self, job_id: str) -> dict:
+        job = self.index.job(job_id)
+        if job is None:
+            raise AccessError(f"unknown job {job_id}")
+        if job["state"] != "running":
+            return {"job_id": job_id, "state": job["state"],
+                    "remaining": len(job["queue"]), "errors": job["errors"]}
+
+        queue: list[str] = job["queue"]
+        errors: list[str] = job["errors"]
+        seen_raw = self.index.meta_get(f"seen:{job_id}")
+        seen: set[str] = set(json.loads(seen_raw)) if seen_raw else set()
+        levels_raw = self.index.meta_get(f"levels:{job_id}")
+        levels: dict[str, int] = (json.loads(levels_raw) if levels_raw else {})
+        warns_raw = self.index.meta_get(f"warns:{job_id}")
+        warns: list[str] = json.loads(warns_raw) if warns_raw else []
+
+        processed = 0
+        while queue:
+            if len(seen) >= MAX_FILES:
+                errors.append(f"file cap reached ({MAX_FILES}) — remaining skipped")
+                queue.clear()
+                break
+            if processed:
+                self._checkpoint(job_id, queue, errors, seen, levels, warns)
+            entry = queue.pop(0)
+            processed += 1
+            try:
+                if entry.startswith("site:"):
+                    kids, w = self._expand_site(entry[5:])
+                    warns.extend(w)
+                    queue.extend(k for k in kids if k not in seen)
+                    continue
+                if entry.startswith("list:"):
+                    _, site_id, list_id = entry.split(":", 2)
+                    for it in _paged(
+                            self.client,
+                            f"sites/{site_id}/lists/{list_id}/items"):
+                        queue.append(f"page:{site_id}:{list_id}:{it['id']}")
+                    continue
+                if entry.startswith("drive:"):
+                    queue.extend(self._expand_drive(entry[6:]))
+                    continue
+                if entry.startswith("page:"):
+                    _, site_id, list_id, item_id = entry.split(":", 3)
+                    page, chunks, w = self._page_from_list_item(
+                        site_id, list_id, item_id)
+                    warns.extend(w)
+                elif entry.startswith("item:"):
+                    _, drive_id, item_id = entry.split(":", 2)
+                    meta = self.client.get(
+                        f"drives/{drive_id}/items/{item_id}").json()
+                    if meta.get("folder") is not None:
+                        queue.extend(self._expand_item(drive_id, item_id))
+                        continue
+                    page, chunks, w = self._page_from_item(
+                        drive_id, item_id, self.downloader)
+                    warns.extend(w)
+                    if page is None:
+                        continue
+                else:
+                    errors.append(f"unknown queue entry {entry}")
+                    continue
+            except AccessError as e:
+                errors.append(f"{entry}: {e}")
+                continue
+            pid = page["page_id"]
+            if pid in seen:
+                continue
+            for c in chunks:
+                c.chunk_id = f"{pid}:{page['content_hash'][:8]}:{c.ordinal}"
+            vecs = None
+            if self.embedder is not None and chunks:
+                vecs = self.embedder.embed(
+                    [f"{c.heading_path}\n{c.text}" for c in chunks])
+            self.index.upsert_page({
+                "page_id": pid, "parent_id": None, "title": page["title"],
+                "url": page["url"], "level": levels.get(pid, 0),
+                "version": page["version"],
+                "content_hash": page["content_hash"],
+                "warnings": page["warnings"]}, chunks, vecs)
+            seen.add(pid)
+
+        self._checkpoint(job_id, queue, errors, seen, levels, warns)
+        self.index.publish(seen)
+        self.index.save_job(job_id, "done" if not queue else "running",
+                            queue, errors)
+        return {"job_id": job_id,
+                "state": "done" if not queue else "running",
+                "remaining": len(queue), "pages_seen": len(seen),
+                "errors": errors, "warnings": warns}

sharepoint_knowledge/server_sp.py

@@ -0,0 +1,334 @@
+"""stdio MCP server for sharepoint-knowledge — the only surface the agent
+reaches. Scope = configured site + signed-in account + chosen subtree."""
+
+from __future__ import annotations
+
+import json
+import logging
+import threading
+
+from confluence_knowledge.client import AccessError, SecurityException
+from confluence_knowledge.index import Index
+from confluence_knowledge.retrieval import (
+    LocalEmbedder,
+    LocalReranker,
+    check_citations,
+    retrieve,
+)
+from confluence_knowledge.settings import SetupError
+from mcp.server.fastmcp import FastMCP
+
+from .auth_sp import acquire_token
+from .client_sp import download_file, make_client
+from .crawl_sp import SPCrawler, resolve_root
+from .settings_sp import APP_NAME, SPProfile, graph_base, home, index_root
+
+mcp = FastMCP("sharepoint-knowledge")
+
+_scopes: dict[str, dict] = {}
+_evidence_sets: dict[str, list[dict]] = {}
+_indexes: dict[str, Index] = {}
+_clients: dict[str, object] = {}
+_EMBEDDER = None
+_RERANKER = None
+_model_lock = threading.Lock()
+_models_ready = threading.Event()
+_warm_lock = threading.Lock()
+_warm_started = False
+
+LOADING_NOTE = "loading local models — call this tool again"
+
+
+def _index_for(scope_key: str) -> Index:
+    idx = _indexes.get(scope_key)
+    if idx is None:
+        idx = _indexes[scope_key] = Index(index_root() / f"{scope_key}.db")
+    return idx
+
+
+def _embedder():
+    global _EMBEDDER
+    with _model_lock:
+        if _EMBEDDER is None:
+            _EMBEDDER = LocalEmbedder(home(), APP_NAME)
+    return _EMBEDDER
+
+
+def _reranker():
+    global _RERANKER
+    with _model_lock:
+        if _RERANKER is None:
+            _RERANKER = LocalReranker(home(), APP_NAME)
+    return _RERANKER
+
+
+def _warm_models() -> None:
+    try:
+        _embedder()
+        _reranker()
+    except SetupError:
+        pass
+    finally:
+        _models_ready.set()
+
+
+def _start_warmup() -> None:
+    global _warm_started
+    with _warm_lock:
+        if _warm_started:
+            return
+        _warm_started = True
+    threading.Thread(target=_warm_models, name="model-warmup", daemon=True).start()
+
+
+def _models_loaded(wait: float) -> bool:
+    _start_warmup()
+    return _models_ready.wait(wait)
+
+
+def _client_for(account: str) -> object:
+    cli = _clients.get(account)
+    if cli is None:
+        profile = SPProfile.load()
+        token = acquire_token(profile, workdir_from_home(), home())
+        cli = _clients[account] = make_client(graph_base(), token)
+    return cli
+
+
+def workdir_from_home():
+    from .settings_sp import workdir
+    return workdir()
+
+
+def _scope(scope_handle: str) -> dict:
+    s = _scopes.get(scope_handle)
+    if s is None:
+        parts = scope_handle.split(":", 3)
+        if len(parts) == 4 and parts[0] == "scope" and \
+                connect(parts[3])["scope_handle"] == scope_handle:
+            return _scopes[scope_handle]
+        raise SetupError("unknown scope — call connect first")
+    return s
+
+
+@mcp.tool()
+def status() -> dict:
+    """Report setup readiness — profile present, token reachable, models."""
+    models = home() / "models"
+    try:
+        profile = SPProfile.load()
+        try:
+            token = acquire_token(profile, workdir_from_home(), home())
+            who = "token ready"
+        except SetupError as e:
+            who = f"sign-in needed: {e}"
+        cfg = {"site": profile.site_url, "tenant": profile.tenant_id,
+               "auth": who}
+        configured = who == "token ready"
+    except SetupError as e:
+        cfg = {"error": str(e)}
+        configured = False
+        token = None
+    if token:
+        try:
+            cli = _client_for(profile.site_url)
+            me = cli.get("me").json()
+            cfg["signed_in_as"] = me.get("userPrincipalName") or me.get("mail")
+        except AccessError as e:
+            cfg["auth"] = str(e)
+            configured = False
+    return {
+        "configured": configured,
+        "profile": cfg,
+        "embedding_model_ready": (models / "all-MiniLM-L6-v2").exists(),
+        "reranker_model_ready": (models / "ms-marco-MiniLM-L6-v2").exists(),
+        "models_loaded": _models_ready.is_set() and _EMBEDDER is not None,
+        "index_root": str(index_root()),
+        "live_scopes": sorted(_scopes),
+    }
+
+
+@mcp.tool()
+def connect(root: str) -> dict:
+    """Bind a scope to a SharePoint URL — a site (pages + all libraries),
+    or a library/folder path inside it. One root per connect."""
+    profile = SPProfile.load()
+    cli = _client_for(profile.site_url)
+    if root.startswith(("site:", "item:", "drive:", "list:", "page:")):
+        root_entry, site_web = root, profile.site_url
+    else:
+        root_entry, site_web = resolve_root(cli, root)
+    principal = "me"
+    scope_key = profile.scope_key(principal, root_entry)
+    idx = _index_for(scope_key)
+    st = idx.stats()
+    handle = f"scope:{scope_key}:{root}"
+    _scopes[handle] = {"key": scope_key, "root_entry": root_entry,
+                       "site_web": site_web, "client": cli}
+    _start_warmup()
+    return {
+        "scope_handle": handle,
+        "root": root_entry,
+        "site": site_web,
+        "cached": st["pages"] > 0,
+        "cached_pages": st["pages"],
+        "cached_as_of": st["as_of"],
+    }
+
+
+_workers: dict[str, threading.Thread] = {}
+_worker_errors: dict[str, str] = {}
+_workers_lock = threading.Lock()
+
+
+def _worker_loop(s: dict, job_id: str) -> None:
+    idx = _index_for(s["key"])
+    try:
+        _models_ready.wait()
+        c = SPCrawler(s["client"], idx, embedder=_embedder(),
+                      downloader=download_file)
+        r = c.step(job_id)
+        while r["state"] == "running":
+            r = c.step(job_id)
+    except Exception as e:  # noqa: BLE001 — surfaced via job state
+        err = f"{type(e).__name__}: {e}"
+        _worker_errors[f"{s['key']}:{job_id}"] = err
+        job = idx.job(job_id)
+        if job and job["state"] == "running":
+            idx.save_job(job_id, "error", job["queue"], job["errors"] + [err])
+
+
+def _ensure_worker(s: dict, job_id: str) -> None:
+    key = f"{s['key']}:{job_id}"
+    with _workers_lock:
+        t = _workers.get(key)
+        if t is not None and t.is_alive():
+            return
+        _workers[key] = threading.Thread(
+            target=_worker_loop, args=(s, job_id),
+            name=f"crawl-{job_id}", daemon=True)
+        _workers[key].start()
+
+
+@mcp.tool()
+def start_crawl(scope_handle: str, refresh: bool = False) -> dict:
+    """Begin indexing the root in the background (Site Pages + libraries)."""
+    s = _scope(scope_handle)
+    idx = _index_for(s["key"])
+    c = SPCrawler(s["client"], idx)
+    job_id = c.start(s["root_entry"])
+    _ensure_worker(s, job_id)
+    return {"job_id": job_id, "state": "running",
+            "note": "crawl runs in the background — call crawl_step to poll"}
+
+
+@mcp.tool()
+def crawl_step(scope_handle: str, job_id: str) -> dict:
+    """Read crawl progress instantly; resumes a dead worker automatically."""
+    s = _scope(scope_handle)
+    idx = _index_for(s["key"])
+    job = idx.job(job_id)
+    if job is None:
+        raise AccessError(f"unknown job {job_id}")
+
+    def progress() -> dict:
+        p = idx.job_progress(job_id)
+        warns = idx.meta_get(f"warns:{job_id}")
+        if warns:
+            p["warnings"] = json.loads(warns)
+        return p
+
+    if job["state"] == "running":
+        if not _models_ready.is_set():
+            _start_warmup()
+            note = LOADING_NOTE
+        else:
+            note = None
+        _ensure_worker(s, job_id)
+        p = progress()
+        if note:
+            p["note"] = note
+        return p
+    p = progress()
+    if p["state"] == "done":
+        p["note"] = "no further calls needed — the index is published"
+    return p
+
+
+def _coverage(idx: Index) -> str:
+    job = idx.latest_job()
+    if job is None or job["state"] == "abandoned":
+        return "unknown"
+    if job["state"] == "running":
+        return "refresh in progress — answering from the previous snapshot"
+    return "partial" if job["errors"] else "complete"
+
+
+@mcp.tool()
+def search(scope_handle: str, question: str) -> dict:
+    """Hybrid retrieve + local rerank; returns bounded, cited evidence."""
+    s = _scope(scope_handle)
+    idx = _index_for(s["key"])
+    st = idx.stats()
+    if st["pages"] == 0:
+        job = idx.latest_job()
+        if job and job["state"] == "running":
+            return {"evidence": [], "coverage": "crawling",
+                    "note": f"crawl {job['job_id']} still running — keep calling crawl_step "
+                            "until state is done, then search"}
+        return {"evidence": [], "coverage": "empty",
+                "note": "nothing indexed — run a crawl first"}
+    if not _models_loaded(10.0):
+        return {"evidence": [], "coverage": "loading", "note": LOADING_NOTE}
+    ev = retrieve(idx, question, _embedder(), _reranker())
+    set_id = f"ev:{len(_evidence_sets)}:{s['key'][:8]}"
+    _evidence_sets[set_id] = ev
+    return {
+        "evidence_set": set_id,
+        "coverage": _coverage(idx),
+        "as_of": st["as_of"],
+        "evidence": [
+            {"chunk_id": e["chunk_id"], "page_title": e["page_title"],
+             "page_url": e["page_url"], "heading": e["heading"],
+             "score": e.get("rerank_score"), "text": e["text"][:2000]}
+            for e in ev
+        ],
+    }
+
+
+@mcp.tool()
+def get_evidence(scope_handle: str, chunk_ids: list[str]) -> dict:
+    """Return full text of previously returned chunks — same scope only."""
+    s = _scope(scope_handle)
+    idx = _index_for(s["key"])
+    all_chunks = {c["chunk_id"]: c for c in idx.active_chunks()}
+    out = []
+    for cid in chunk_ids[:16]:
+        c = all_chunks.get(cid)
+        if c is None:
+            raise SecurityException(f"chunk {cid} not in this scope's index")
+        p = idx.active_pages().get(c["page_id"], {})
+        out.append({"chunk_id": cid, "text": c["text"],
+                    "page_title": p.get("title", ""), "page_url": p.get("url", "")})
+    return {"chunks": out}
+
+
+@mcp.tool()
+def verify_citations(scope_handle: str, evidence_set: str,
+                     mappings: list[dict]) -> dict:
+    _scope(scope_handle)
+    ev = _evidence_sets.get(evidence_set)
+    if ev is None or not evidence_set.endswith(_scopes[scope_handle]["key"][:8]):
+        raise SecurityException(
+            "unknown or cross-scope evidence set — run search again and cite its evidence_set")
+    return check_citations(ev, mappings)
+
+
+def main() -> None:
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    _start_warmup()
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

sharepoint_knowledge/settings_sp.py

@@ -0,0 +1,80 @@
+"""Filesystem layout + connection profile for sharepoint-knowledge.
+
+Same machinery as the other connectors: ~/.sharepoint-knowledge home,
+SP_KNOWLEDGE_* env overrides, a non-secret profile.json. The token cache
+itself is a credential blob in the OS keyring (or protected-file fallback).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from confluence_knowledge.settings import (
+    SetupError,
+    default_home,
+    resolve_workdir,
+)
+
+APP_NAME = "sharepoint-knowledge"
+INDEX_DIR_NAME = ".sp_knowledge_index"
+PROFILE_NAME = "profile.json"
+AUTH_FILE_NAME = ".sp_auth.json"
+AUTH_ENV = "SP_KNOWLEDGE_TOKEN"
+
+
+def home() -> Path:
+    return default_home("SP_KNOWLEDGE_HOME", ".sharepoint-knowledge")
+
+
+def workdir() -> Path:
+    return resolve_workdir("SP_KNOWLEDGE_WORKDIR",
+                           "SP_KNOWLEDGE_HOME", ".sharepoint-knowledge")
+
+
+def index_root() -> Path:
+    return workdir() / INDEX_DIR_NAME
+
+
+def graph_base() -> str:
+    """Microsoft Graph base — overridable for the fake server in e2e tests."""
+    return os.environ.get("SP_KNOWLEDGE_GRAPH_BASE",
+                          "https://graph.microsoft.com/v1.0")
+
+
+@dataclass(frozen=True)
+class SPProfile:
+    """Non-secret connection profile. The token cache is a credential blob."""
+
+    tenant_id: str
+    client_id: str
+    site_url: str       # canonical https site URL the user indexed
+    credential_ref: str # opaque key — not the token
+
+    def scope_key(self, principal: str, root: str) -> str:
+        raw = f"{self.site_url}|{principal}|{root}"
+        return hashlib.sha256(raw.encode()).hexdigest()[:24]
+
+    def save(self) -> Path:
+        p = home() / PROFILE_NAME
+        p.parent.mkdir(parents=True, exist_ok=True)
+        tmp = p.with_suffix(".tmp")
+        tmp.write_text(json.dumps({
+            "tenant_id": self.tenant_id, "client_id": self.client_id,
+            "site_url": self.site_url, "credential_ref": self.credential_ref,
+        }, indent=2))
+        os.replace(tmp, p)
+        return p
+
+    @staticmethod
+    def load() -> SPProfile:
+        p = home() / PROFILE_NAME
+        if not p.exists():
+            raise SetupError("no SharePoint profile configured — run "
+                             "`sharepoint-knowledge setup`")
+        d = json.loads(p.read_text())
+        return SPProfile(d["tenant_id"], d["client_id"], d["site_url"],
+                         d["credential_ref"])

sharepoint_knowledge-0.1.0.dist-info/METADATA

@@ -0,0 +1,40 @@
+Metadata-Version: 2.4
+Name: sharepoint-knowledge
+Version: 0.1.0
+Summary: SharePoint connector for the grounded-Q&A engine — site pages + document libraries, Entra device-code sign-in, local models, cited answers
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/nkunal1992/agents
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: confluence-knowledge[files]>=0.4.1
+Requires-Dist: msal>=1.31
+Requires-Dist: httpx>=0.27
+Requires-Dist: mcp>=1.8
+Provides-Extra: models
+Requires-Dist: confluence-knowledge[models]>=0.4.1; extra == "models"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: ruff>=0.5; extra == "dev"
+Requires-Dist: python-docx>=1.1; extra == "dev"
+Requires-Dist: python-pptx>=1.0; extra == "dev"
+Requires-Dist: openpyxl>=3.1; extra == "dev"
+Requires-Dist: pypdf>=5.0; extra == "dev"
+
+# sharepoint-knowledge
+
+SharePoint connector on the shared grounded-Q&A engine — Site Pages +
+document libraries (docx/pdf/pptx/xlsx/csv/txt/md/html), Entra device-code
+sign-in, local models, cited answers. Same engine as
+[`confluence-knowledge`](../confluence-knowledge) and
+[`local-knowledge`](../local-knowledge).
+
+- `sharepoint_knowledge/auth_sp.py` — MSAL device-code flow, token cache in
+  the OS keyring (protected-file fallback), silent refresh
+- `sharepoint_knowledge/client_sp.py` — GET-only Graph client; downloads go
+  through `@microsoft.graph.downloadUrl` WITHOUT the Bearer header
+- `sharepoint_knowledge/crawl_sp.py` — durable BFS site → pages + libraries
+- `sharepoint_knowledge/server_sp.py` — the `sharepoint-knowledge/*` tools
+- `sharepoint_knowledge/cli_sp.py` — setup/doctor/prepare-models/register/
+  update/serve/clean
+
+See `docs/sharepoint-knowledge.md` for the full guide.

sharepoint_knowledge-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+sharepoint_knowledge/__init__.py,sha256=J8KcyQSaE9FuCWplCAnhlSJW0DNI48H4oDqCM8gxiUo,97
+sharepoint_knowledge/auth_sp.py,sha256=vzschQ7NEL8XYb2LnjMkEFKh7Pu5h0OSQVyqQSJ7_Fk,4314
+sharepoint_knowledge/cli_sp.py,sha256=n7iFP-MKg2QQp6NtJGL9ponUw-9ll2fIYzpokZrSG0k,7153
+sharepoint_knowledge/client_sp.py,sha256=ZvRWSuz0NATPgS8JH_WTO-ef17kCXNu8odH3JSsacOU,3055
+sharepoint_knowledge/crawl_sp.py,sha256=7t90HE_cCl5pLIRKYhrLo1Wqg3h-SUIoLEO4fCT39aY,12242
+sharepoint_knowledge/server_sp.py,sha256=VueieIIAiVXmpZ0N0j_IvY-XI5oq6P8jrYhfz_ppI_g,10601
+sharepoint_knowledge/settings_sp.py,sha256=Coz78N3VpmulKyvHUpTdiAdzd1NODx581qsMfXDDxjQ,2483
+sharepoint_knowledge-0.1.0.dist-info/METADATA,sha256=xZsG0XWvi8pGVj_yCZ5tdy4imqvfjusbAqorfZ9RP0E,1779
+sharepoint_knowledge-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+sharepoint_knowledge-0.1.0.dist-info/entry_points.txt,sha256=Hwjazt5G5FCCo1vQLJMl6WcHA6Ss-Ux4kIob2bLlj_I,74
+sharepoint_knowledge-0.1.0.dist-info/top_level.txt,sha256=LKQ769avV5sRwxHMNtEe5bTs2HmV79KZ3to22ymUBX8,21
+sharepoint_knowledge-0.1.0.dist-info/RECORD,,

sharepoint_knowledge-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sharepoint_knowledge-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+sharepoint-knowledge = sharepoint_knowledge.cli_sp:main

sharepoint_knowledge-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+sharepoint_knowledge