confluence-knowledge 0.3.1__py3-none-any.whl

confluence_knowledge/__init__.py

@@ -0,0 +1,3 @@
+"""confluence-knowledge: local GET-only Confluence subtree index + grounded Q&A."""
+
+__version__ = "0.1.0"

confluence_knowledge/auth.py

@@ -0,0 +1,83 @@
+"""Credential storage: OS keyring → env var → protected file (explicit choice).
+
+Tokens are never accepted via chat, MCP tool arguments, or command-line flags —
+the setup wizard prompts with getpass in the user's own terminal.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+from pathlib import Path
+
+from .settings import APP_NAME, AUTH_FILE_NAME, ensure_gitignore, reject_unsafe_path
+
+SERVICE = APP_NAME
+ENV_TOKEN = "CONF_KNOWLEDGE_TOKEN"
+ENV_EMAIL = "CONF_KNOWLEDGE_EMAIL"
+
+
+def _keyring_store(ref: str, secret: str) -> bool:
+    try:
+        import keyring
+        from keyring.backends.fail import Keyring as FailBackend
+        if isinstance(keyring.get_keyring(), FailBackend):
+            return False
+        keyring.set_password(SERVICE, ref, secret)
+        return True
+    except Exception:  # noqa: BLE001 — keyring backends fail in many ways; fallback is intentional
+        return False
+
+
+def _keyring_get(ref: str) -> str | None:
+    try:
+        import keyring
+        return keyring.get_password(SERVICE, ref)
+    except Exception:  # noqa: BLE001 — unavailable keyring → try next provider
+        return None
+
+
+def store_credential(workdir: Path, home: Path, ref: str, token: str) -> str:
+    """Persist token: keyring preferred, protected file as explicit fallback.
+
+    Returns a human description of where it went (never the token).
+    """
+    if _keyring_store(ref, token):
+        return "OS credential store"
+    # file fallback — gitignore BEFORE writing
+    ensure_gitignore(workdir, [AUTH_FILE_NAME, ".conf_knowledge_index/"])
+    p = reject_unsafe_path(workdir / AUTH_FILE_NAME)
+    data = json.loads(p.read_text()) if p.exists() else {}
+    data[ref] = token
+    tmp = p.with_suffix(".tmp")
+    tmp.write_text(json.dumps(data))
+    os.replace(tmp, p)
+    if os.name != "nt":
+        os.chmod(p, stat.S_IRUSR | stat.S_IWUSR)
+    return f"protected file {p} (chmod 600)"
+
+
+def get_credential(workdir: Path, ref: str) -> str | None:
+    """Lookup order: env var (CI/scripts) → keyring → protected file."""
+    t = os.environ.get(ENV_TOKEN)
+    if t:
+        return t
+    t = _keyring_get(ref)
+    if t:
+        return t
+    p = workdir / AUTH_FILE_NAME
+    if p.exists():
+        data = json.loads(p.read_text())
+        return data.get(ref)
+    p2 = Path.home() / ".confluence-knowledge" / AUTH_FILE_NAME
+    if p2.exists():
+        return json.loads(p2.read_text()).get(ref)
+    return None
+
+
+def redact(text: str, secrets: list[str]) -> str:
+    for s in secrets:
+        if s:
+            text = text.replace(s, "***")
+    return text

confluence_knowledge/cli.py

@@ -0,0 +1,180 @@
+"""confluence-knowledge CLI: setup / doctor / prepare-models / serve / clean.
+
+The setup wizard prompts for tokens via getpass in the user's own terminal —
+tokens never travel through chat or command-line arguments.
+"""
+
+from __future__ import annotations
+
+import argparse
+import getpass
+import json
+import os
+import sys
+from pathlib import Path
+
+from .auth import store_credential
+from .client import AccessError, make_client
+from .retrieval import prepare_models
+from .settings import (
+    AUTH_FILE_NAME,
+    INDEX_DIR_NAME,
+    SetupError,
+    SiteProfile,
+    default_home,
+    ensure_gitignore,
+    resolve_workdir,
+)
+
+
+def _wizard() -> int:
+    print("confluence-knowledge setup\n")
+    env = ""
+    while env not in ("1", "2"):
+        env = input("Confluence environment — [1] Cloud (atlassian.net) "
+                    "or [2] Data Center (corporate URL)? ").strip()
+    env = "cloud" if env == "1" else "datacenter"
+
+    if env == "cloud":
+        print("\nCreate a token: sign in to https://id.atlassian.com → "
+              "Security → Create and manage API tokens → Create API token "
+              "(choose Confluence read scopes if scoped tokens are offered).")
+        site = input("\nSite URL (e.g. https://acme.atlassian.net): ").strip()
+        email = input("Atlassian account email: ").strip()
+    else:
+        print("\nCreate a token: Confluence → your profile picture → Settings → "
+              "Personal Access Tokens → Create token (read access is enough).")
+        site = input("\nSite URL incl. context path (e.g. https://conf.acme.corp/confluence): ").strip()
+        email = None
+
+    try:
+        base = SiteProfile.canonical_base(env, site)
+    except SetupError as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+
+    workdir = resolve_workdir()
+    workdir.mkdir(parents=True, exist_ok=True)
+    home = default_home()
+    # gitignore protections BEFORE any secret/cache write
+    try:
+        ensure_gitignore(workdir, [AUTH_FILE_NAME, ".conf_knowledge_index/"])
+    except SetupError as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+
+    ref = f"{env}:{base}:{email or '-'}"
+    token = getpass.getpass("Token (input hidden): ")
+    if not token:
+        print("error: empty token", file=sys.stderr)
+        return 2
+
+    where = store_credential(workdir, home, ref, token)
+    SiteProfile(env, base, email, ref).save(home)
+
+    # heartbeat
+    try:
+        c = make_client(SiteProfile(env, base, email, ref), token)
+        ep = "wiki/rest/api/user/current" if env == "cloud" else "rest/api/user/current"
+        who = c.get(ep).json()
+        print(f"\nconnected as: {who.get('displayName') or who.get('username') or 'OK'}")
+    except Exception as e:  # noqa: BLE001 — any connect failure (auth/network/TLS) exits non-zero
+        print(f"\nconnection check failed: {e}", file=sys.stderr)
+        return 3
+
+    print(f"credential stored: {where}")
+    print(f"profile: {home / 'profile.json'}")
+    print("\nNext: `confluence-knowledge prepare-models` to fetch the local "
+          "models (Apache-2.0), then point your IDE's MCP config at:\n"
+          f"  command: {sys.executable}\n"
+          f"  args:    [\"-m\", \"confluence_knowledge.server\"]\n"
+          f"  env:     CONF_KNOWLEDGE_WORKDIR={workdir}")
+    return 0
+
+
+def _doctor() -> int:
+    from .server import status
+    try:
+        s = status()
+        print(json.dumps(s, indent=2))
+        if s.get("configured"):
+            from .server import _client_for  # internal but fine in CLI
+            profile = SiteProfile.load(default_home())
+            try:
+                c = _client_for(profile)
+                ep = ("wiki/rest/api/user/current" if profile.env == "cloud"
+                      else "rest/api/user/current")
+                who = c.get(ep).json()
+                print("heartbeat: OK —",
+                      who.get("displayName") or who.get("username"))
+            except (AccessError, SetupError) as e:
+                print(f"heartbeat: FAILED — {e}", file=sys.stderr)
+                return 3
+        return 0
+    except SetupError as e:
+        print(f"setup error: {e}", file=sys.stderr)
+        return 2
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(prog="confluence-knowledge")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("setup", help="guided credential + config setup")
+    sub.add_parser("doctor", help="verify config, credentials, models")
+    sub.add_parser("prepare-models", help="download the pinned local models")
+    reg = sub.add_parser(
+        "register",
+        help="write this server into the IDE's MCP config (no manual paste)")
+    reg.add_argument("--ide", default="auto",
+                     choices=["auto", "all", "vscode", "jetbrains"],
+                     help="which config files to write (default: all that apply)")
+    reg.add_argument("--workspace", type=Path, default=None,
+                     help="also write <workspace>/.vscode/mcp.json")
+    reg.add_argument("--remove", action="store_true",
+                     help="remove our entry instead of adding it")
+    sub.add_parser("serve", help="run the stdio MCP server")
+    clean = sub.add_parser("clean", help="delete the local index for this repo")
+    clean.add_argument("--yes", action="store_true")
+    args = ap.parse_args()
+
+    if args.cmd == "setup":
+        sys.exit(_wizard())
+    if args.cmd == "doctor":
+        sys.exit(_doctor())
+    if args.cmd == "prepare-models":
+        for line in prepare_models(default_home()):
+            print(line)
+        return
+    if args.cmd == "register":
+        from .registration import register
+        try:
+            wd = (Path(os.environ["CONF_KNOWLEDGE_WORKDIR"]).expanduser().resolve()
+                  if os.environ.get("CONF_KNOWLEDGE_WORKDIR") else None)
+            results = register(ide=args.ide, workspace=args.workspace,
+                               workdir=wd, remove=args.remove)
+        except SetupError as e:
+            print(f"error: {e}", file=sys.stderr)
+            sys.exit(2)
+        for path, action in results:
+            print(f"{action}: {path}")
+        if not args.remove:
+            print("\nReload your IDE so Copilot picks up the server, "
+                  "then re-ask the agent.")
+        return
+    if args.cmd == "serve":
+        from .server import main as serve
+        serve()
+        return
+    if args.cmd == "clean":
+        import shutil
+
+        target = resolve_workdir() / INDEX_DIR_NAME
+        if target.exists() and (args.yes or input(
+                f"delete {target}? [y/N] ").strip().lower() == "y"):
+            shutil.rmtree(target)
+            print("index deleted")
+        return
+
+
+if __name__ == "__main__":
+    main()

confluence_knowledge/client.py

@@ -0,0 +1,152 @@
+"""GET-only Confluence REST client.
+
+Every transport call passes through request(): a non-GET method is rejected
+with SecurityException before any network I/O, and paths are validated against
+a per-adapter endpoint allowlist. No convenience method can bypass this.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from urllib.parse import urljoin, urlparse
+
+import httpx
+
+
+class SecurityException(RuntimeError):
+    """Raised when a write/delete/mutation HTTP operation is attempted."""
+
+
+class AccessError(RuntimeError):
+    """Authentication, permission, or reachability failure (actionable)."""
+
+
+@dataclass(frozen=True)
+class EndpointRule:
+    pattern: re.Pattern  # matched against the API path
+    purpose: str
+
+
+class ConfluenceClient:
+    """GET-only client bound to one site base URL and one route allowlist."""
+
+    def __init__(
+        self,
+        base_url: str,
+        headers: dict[str, str],
+        rules: list[EndpointRule],
+        timeout: float = 20.0,
+        max_bytes: int = 8 * 1024 * 1024,
+        session: httpx.Client | None = None,
+    ):
+        u = urlparse(base_url)
+        if u.scheme != "https" and u.hostname not in ("localhost", "127.0.0.1"):
+            raise AccessError("Confluence base URL must be https")
+        self._base = base_url.rstrip("/") + "/"
+        self._base_origin = f"{u.scheme}://{u.netloc}"
+        self._base_path = u.path.rstrip("/")
+        self._rules = rules
+        self._http = session or httpx.Client(
+            headers={"Accept": "application/json", **headers},
+            timeout=timeout,
+            follow_redirects=False,  # never carry credentials through redirects
+        )
+        self._max_bytes = max_bytes
+
+    # ---- the GET-only wrapper ----
+
+    def request(self, method: str, path: str, params: dict | None = None) -> httpx.Response:
+        if method.upper() != "GET":
+            raise SecurityException(
+                "Error: This agent is strictly read-only. "
+                "Write/Delete operations are forbidden."
+            )
+        api_path = self._validate_path(path)
+        resp = self._http.get(urljoin(self._base, api_path.lstrip("/")), params=params)
+        if resp.status_code >= 400:
+            raise AccessError(self._describe(resp))
+        if resp.status_code >= 300:
+            raise AccessError(
+                f"unexpected redirect ({resp.status_code}) — check the base URL/context path"
+            )
+        if len(resp.content) > self._max_bytes:
+            raise AccessError(f"response exceeds {self._max_bytes} bytes — refusing to buffer")
+        return resp
+
+    def get(self, path: str, params: dict | None = None) -> httpx.Response:
+        return self.request("GET", path, params)
+
+    # ---- validation ----
+
+    def _validate_path(self, path: str) -> str:
+        """Normalize to an API-relative path; enforce site + endpoint bounds."""
+        u = urlparse(path)
+        if u.scheme or u.netloc:
+            # absolute URL (e.g. pagination next-link) must stay inside
+            # the configured origin + API base path
+            full_base = self._base_origin + self._base_path
+            full = f"{u.scheme}://{u.netloc}{u.path}"
+            if not (full == full_base or full.startswith(full_base + "/")):
+                raise SecurityException(
+                    "Error: request escapes the configured Confluence site — refused."
+                )
+            rel = u.path[len(self._base_path):].lstrip("/")
+        elif path.startswith("/"):
+            # site-relative path must live under the API base path
+            if self._base_path and not path.startswith(self._base_path + "/"):
+                raise SecurityException(
+                    "Error: request escapes the configured Confluence site — refused."
+                )
+            rel = path[len(self._base_path):].lstrip("/")
+        else:
+            rel = path
+        if ".." in rel:
+            raise SecurityException("Error: path traversal rejected.")
+        if not any(r.pattern.match(rel) for r in self._rules):
+            raise SecurityException(
+                "Error: This agent is strictly read-only. "
+                f"Endpoint not on the allowed GET list: {rel}"
+            )
+        return rel
+
+    @staticmethod
+    def _describe(resp: httpx.Response) -> str:
+        s = resp.status_code
+        return {
+            401: "authentication failed (401) — check the token and account",
+            403: "permission denied (403) — your account can't view this content",
+            404: "not found (404) — check the page ID/space and base URL",
+            429: "rate limited (429) — retry after the server's Retry-After",
+        }.get(s, f"HTTP {s} from Confluence")
+
+
+# ---- adapters ----
+
+RULES_V2 = [
+    EndpointRule(re.compile(r"wiki/api/v2/pages/\d+/?$"), "page body"),
+    EndpointRule(re.compile(r"wiki/api/v2/pages/\d+/direct-children/?$"), "children"),
+    EndpointRule(re.compile(r"wiki/api/v2/folders/\d+/direct-children/?$"), "folder children"),
+    EndpointRule(re.compile(r"wiki/rest/api/user/current/?$"), "identity"),
+]
+
+RULES_DC = [
+    EndpointRule(re.compile(r"(?:confluence/)?rest/api/content/\d+/?$"), "page body"),
+    EndpointRule(re.compile(r"(?:confluence/)?rest/api/content/\d+/child/page/?$"), "children"),
+    EndpointRule(re.compile(r"(?:confluence/)?rest/api/user/current/?$"), "identity"),
+]
+
+def make_client(profile, token: str, cloud_id: str | None = None) -> ConfluenceClient:
+    """Build a client for the configured profile. Token only in headers."""
+    import base64
+
+    if profile.env == "cloud":
+        auth = base64.b64encode(f"{profile.email}:{token}".encode()).decode()
+        # scoped tokens go through the Atlassian gateway bound to one cloudId;
+        # the same v2 route shapes apply relative to that base path
+        base = (f"https://api.atlassian.com/ex/confluence/{cloud_id}"
+                if cloud_id else profile.base_url)
+        return ConfluenceClient(base, {"Authorization": f"Basic {auth}"}, RULES_V2)
+    return ConfluenceClient(
+        profile.base_url, {"Authorization": f"Bearer {token}"}, RULES_DC
+    )

confluence_knowledge/crawl.py

@@ -0,0 +1,236 @@
+"""Durable, resumable breadth-first crawl of one page subtree.
+
+Queue and progress live in the index DB — a crawl can be interrupted at any
+point and resumed by job ID. Bodies are fetched only for pages admitted to the
+root's descendant graph.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+import uuid
+from dataclasses import dataclass, field
+from urllib.parse import urlparse
+
+from .client import AccessError, ConfluenceClient
+from .extract import extract_chunks
+from .index import Index
+
+
+@dataclass
+class CrawlJob:
+    job_id: str
+    root_page: str
+    queue: list[str] = field(default_factory=list)
+    visited: set[str] = field(default_factory=set)
+    errors: list[str] = field(default_factory=list)
+
+
+def _fetch_children_v2(client: ConfluenceClient, page_id: str) -> list[tuple[str, str]]:
+    """(child_id, child_type) — pages and folders both; folders are traversed
+    but never indexed as content."""
+    out, cursor = [], None
+    while True:
+        params = {"limit": 250}
+        if cursor:
+            params["cursor"] = cursor
+        r = client.get(f"wiki/api/v2/pages/{page_id}/direct-children", params)
+        d = r.json()
+        for ch in d.get("results", []):
+            out.append((str(ch["id"]), ch.get("type", "page")))
+        nxt = (d.get("_links") or {}).get("next")
+        if not nxt:
+            break
+        cursor = nxt.split("cursor=")[-1].split("&")[0]
+    return out
+
+
+def _fetch_folder_children_v2(client: ConfluenceClient, folder_id: str) -> list[tuple[str, str]]:
+    out, cursor = [], None
+    while True:
+        params = {"limit": 250}
+        if cursor:
+            params["cursor"] = cursor
+        r = client.get(f"wiki/api/v2/folders/{folder_id}/direct-children", params)
+        d = r.json()
+        for ch in d.get("results", []):
+            out.append((str(ch["id"]), ch.get("type", "page")))
+        nxt = (d.get("_links") or {}).get("next")
+        if not nxt:
+            break
+        cursor = nxt.split("cursor=")[-1].split("&")[0]
+    return out
+
+
+def _fetch_children_dc(client: ConfluenceClient, page_id: str) -> list[tuple[str, str]]:
+    out, start = [], 0
+    while True:
+        r = client.get(f"rest/api/content/{page_id}/child/page",
+                       params={"limit": 200, "start": start})
+        d = r.json()
+        for ch in d.get("results", []):
+            out.append((str(ch["id"]), "page"))
+        if d.get("size", 0) < d.get("limit", 200):
+            break
+        start += d["size"]
+    return out
+
+
+def _abs_url(webui: str, base: str | None) -> str:
+    """Join Confluence's relative `_links.webui` onto a base so citations are
+    clickable; absolute links pass through unchanged."""
+    if not webui or urlparse(webui).scheme or not base:
+        return webui
+    return base.rstrip("/") + "/" + webui.lstrip("/")
+
+
+def _fetch_page_v2(client: ConfluenceClient, page_id: str) -> dict:
+    d = client.get(f"wiki/api/v2/pages/{page_id}",
+                   params={"body-format": "storage"}).json()
+    links = d.get("_links") or {}
+    return {
+        "page_id": str(d["id"]), "title": d.get("title", ""),
+        "storage": (d.get("body") or {}).get("storage", {}).get("value", ""),
+        "version": str((d.get("version") or {}).get("number", "")),
+        "url": _abs_url(links.get("webui", ""), links.get("base")),
+    }
+
+
+def _fetch_page_dc(client: ConfluenceClient, page_id: str) -> dict:
+    d = client.get(f"rest/api/content/{page_id}",
+                   params={"expand": "body.storage,version,ancestors"}).json()
+    links = d.get("_links") or {}
+    return {
+        "page_id": str(d["id"]), "title": d.get("title", ""),
+        "storage": (d.get("body") or {}).get("storage", {}).get("value", ""),
+        "version": str((d.get("version") or {}).get("number", "")),
+        "url": _abs_url(links.get("webui", ""), links.get("base")),
+        "ancestors": [str(a["id"]) for a in d.get("ancestors", [])],
+    }
+
+
+class Crawler:
+    def __init__(self, client: ConfluenceClient, index: Index, env: str,
+                 embedder=None, site_url: str | None = None):
+        self.client = client
+        self.index = index
+        self.env = env
+        self.embedder = embedder  # retrieval.LocalEmbedder or None
+        # fallback base for page links when a response carries no _links.base
+        if site_url and env == "cloud" and not site_url.rstrip("/").endswith("/wiki"):
+            site_url = site_url.rstrip("/") + "/wiki"
+        self.site_url = site_url
+
+    def _children(self, pid: str, is_folder: bool = False):
+        if self.env == "cloud":
+            return (_fetch_folder_children_v2(self.client, pid) if is_folder
+                    else _fetch_children_v2(self.client, pid))
+        return _fetch_children_dc(self.client, pid)
+
+    def _page(self, pid: str) -> dict:
+        page = (_fetch_page_v2(self.client, pid) if self.env == "cloud"
+                else _fetch_page_dc(self.client, pid))
+        page["url"] = _abs_url(page["url"], self.site_url)
+        return page
+
+    def start(self, root_page: str) -> str:
+        job_id = uuid.uuid4().hex[:12]
+        self.index.abandon_running_jobs()
+        self.index.create_job(job_id, root_page)
+        self.index.save_job(job_id, "running", [f"page:{root_page}"], [])
+        return job_id
+
+    def _checkpoint(self, job_id, queue, errors, seen, levels) -> None:
+        """Persist progress so an interrupted step loses no completed page."""
+        with self.index.lock, self.index.db:
+            self.index.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)",
+                                  (f"seen:{job_id}", json.dumps(sorted(seen))))
+            self.index.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)",
+                                  (f"levels:{job_id}", json.dumps(levels)))
+            self.index.db.execute(
+                "UPDATE jobs SET queue=?, errors=? WHERE job_id=?",
+                (json.dumps(queue), json.dumps(errors), job_id))
+
+    def step(self, job_id: str, batch: int | None = 10,
+             budget_s: float | None = None) -> dict:
+        """Process up to `batch` pages (None = whole queue), stopping early
+        once `budget_s` seconds have passed. The background worker calls this
+        unbounded; a bounded call stays under the IDE's request timeout."""
+        deadline = None if budget_s is None else time.monotonic() + budget_s
+        job = self.index.job(job_id)
+        if job is None:
+            raise AccessError(f"unknown job {job_id}")
+        if job["state"] != "running":
+            return {"job_id": job_id, "state": job["state"],
+                    "remaining": len(job["queue"]), "errors": job["errors"]}
+
+        queue: list[str] = job["queue"]
+        errors: list[str] = job["errors"]
+        seen_raw = self.index.meta_get(f"seen:{job_id}")
+        seen: set[str] = set(json.loads(seen_raw)) if seen_raw else set()
+        levels_raw = self.index.meta_get(f"levels:{job_id}")
+        levels: dict[str, int] = (json.loads(levels_raw) if levels_raw
+                                  else {job["root_page"]: 0})
+        visited: set[str] = set()
+
+        processed = 0
+        while queue and (batch is None or processed < batch) and (
+                deadline is None or processed == 0 or time.monotonic() < deadline):
+            if processed:
+                self._checkpoint(job_id, queue, errors, seen, levels)
+            entry = queue.pop(0)          # "page:123" or "folder:456"
+            is_folder = entry.startswith("folder:")
+            pid = entry.split(":", 1)[1] if ":" in entry else entry
+            if pid in visited or pid in seen:
+                continue
+            visited.add(pid)
+            processed += 1
+            if is_folder:
+                # traverse folders to reach descendant pages — never indexed
+                for cid, ctype in self._children(pid, is_folder=True):
+                    if cid not in seen:
+                        queue.append(f"{ctype}:{cid}")
+                continue
+            try:
+                page = self._page(pid)
+            except AccessError as e:
+                errors.append(f"page {pid}: {e}")
+                continue
+            if pid != job["root_page"]:
+                anc = page.get("ancestors") or []
+                if anc and job["root_page"] not in anc:
+                    errors.append(f"page {pid}: outside root subtree — skipped")
+                    continue
+            content_hash = hashlib.sha256(
+                page["storage"].encode()).hexdigest()
+            chunks, warns = extract_chunks(page["storage"])
+            for c in chunks:
+                c.chunk_id = f"{pid}:{content_hash[:8]}:{c.ordinal}"
+            vecs = None
+            if self.embedder is not None and chunks:
+                vecs = self.embedder.embed(
+                    [f"{c.heading_path}\n{c.text}" for c in chunks])
+            self.index.upsert_page({
+                "page_id": pid, "parent_id": None, "title": page["title"],
+                "url": page["url"], "level": levels.get(pid, 0),
+                "version": page["version"], "content_hash": content_hash,
+                "warnings": warns}, chunks, vecs)
+            seen.add(pid)
+            for cid, ctype in self._children(pid, is_folder=False):
+                if cid not in seen:
+                    levels.setdefault(cid, levels.get(pid, 0) + 1)
+                    queue.append(f"{ctype}:{cid}")
+
+        self._checkpoint(job_id, queue, errors, seen, levels)
+
+        if queue:
+            self.index.save_job(job_id, "running", queue, errors)
+            state = "running"
+        else:
+            self.index.publish(seen)
+            self.index.save_job(job_id, "done", [], errors)
+            state = "done"
+        return {"job_id": job_id, "state": state, "remaining": len(queue),
+                "pages_seen": len(seen), "errors": errors}