webbee 0.6.18__tar.gz → 0.7.0__tar.gz

{webbee-0.6.18 → webbee-0.7.0}/CHANGELOG.md

@@ -1,5 +1,42 @@
 # Changelog
 
+## 0.7.0
+
+* **The local index takes any folder and every file kind**: open webbee in a home folder, a folder of contracts or a
+  Kotlin app - code in 20+ languages, documents (DOCX, PDF, ODT, legacy DOC, Markdown, HTML), sheets (XLSX, CSV),
+  slides, mail (EML, MBOX) and image metadata are indexed. The folder you work in is indexed first, then what changed
+  in the last 30 days, then the rest; a big folder builds in a process pool.
+* **Only code leaves the machine**: what the vault may get is unchanged - code and resolution configs. Documents, mail
+  and sheets stay in the local index.
+* **Never read**: secrets by name, keychains, `.gnupg`, browser profiles, wallets, VPN and cloud credential files, iCloud
+  placeholders (reading one would download it) and app packages (`.photoslibrary`, `.app`). A secret written later
+  is never read either.
+* **Changes found by checksums, not by rescanning**: every indexed folder has a checksum tree - one hash per directory
+  over the size and time of what is inside (`webbee.intel.tree`). A start from the cache and every refresh compare the
+  trees top down and skip an unchanged directory whole; a git repository is checked through git (HEAD, `git status`
+  and the stats of the files it names) with no walk at all. Measured on a folder of 7 repositories (1862 files): a
+  start from the cache 0.3 s, a refresh with nothing changed 0.17 s, one edited file found in 0.21 s comparing 19
+  directories. `IntelService.load_or_build()`, `refresh()`, `checksum()`.
+* **One listing** for the build, the cache check and every refresh (`_index_listing`): a `.gitignore`'d file of a
+  repository is never read, links and anything but a regular file are skipped.
+* The cache keeps each file's kind (a document was read back as code after a restart) and a folder without git is
+  cached too (it was indexed from scratch at every start). Cache schema 5.
+* A stuck file never stops a build: documents and big sets are built in a process pool of at most 4 processes; a job
+  that runs past its time (or whose process dies) is killed and its files retried one by one, a file still stuck is
+  skipped and counted as `timeout` in `progress()`.
+* **A server's code is read on the server**: `remote_read` (a file's lines, at most 2 MB, never a secret by its
+  name or where a link leads, never outside the repo) and `remote_grep` (`git grep` there, else `grep -r`; code files
+  only) over your own ssh (`BatchMode`, never a prompt). The vault holds the code index, never the code.
+* **Login survives a non-JSON answer**: the device login's token poll crashed on a balancer's page
+  (`JSONDecodeError`); it now keeps polling (the fix lives in `imperal-mcp` 0.9.0, now the floor).
+* Needs `imperal-sdk` 6.4.1 with its `universal-index` extra and `imperal-mcp` 0.9.0 (pulled in).
+
+## 0.6.19
+
+* **Another agent's message to a running session stays on screen**: it printed only as a queue-panel row that left the
+  moment the turn took it (seconds) -- Claude's follow-ups to a running marathon were never seen. It now also prints
+  one kept line, `✉ [agent] Claude (imperal-mcp): ...`, once per message.
+
 ## 0.6.18
 
 * **A file is read whole -- up to 5000 lines**: a window asked of a file that fits answers with the whole file (the

{webbee-0.6.18 → webbee-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: webbee
-Version: 0.6.18
+Version: 0.7.0
 Summary: Webbee 🐝 — the Imperal Cloud coding agent in your terminal
 Project-URL: Homepage, https://imperal.io
 Project-URL: Documentation, https://docs.imperal.io
@@ -31,8 +31,8 @@ Requires-Python: >=3.7
 Requires-Dist: asyncssh>=2.14; python_version >= '3.10'
 Requires-Dist: httpx-sse>=0.4; python_version >= '3.10'
 Requires-Dist: httpx>=0.27; python_version >= '3.10'
-Requires-Dist: imperal-mcp>=0.8.12; python_version >= '3.10'
-Requires-Dist: imperal-sdk>=6.3.6; python_version >= '3.10'
+Requires-Dist: imperal-mcp>=0.9.0; python_version >= '3.10'
+Requires-Dist: imperal-sdk[universal-index]>=6.4.1; python_version >= '3.10'
 Requires-Dist: model2vec>=0.8.2; python_version >= '3.10' and (sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13')
 Requires-Dist: numpy>=2; python_version >= '3.10' and (sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13')
 Requires-Dist: prompt-toolkit>=3; python_version >= '3.10'

{webbee-0.6.18 → webbee-0.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "webbee"
-version = "0.6.18"
+version = "0.7.0"
 description = "Webbee 🐝 — the Imperal Cloud coding agent in your terminal"
 readme = "README.md"
 license = "GPL-3.0-or-later"
@@ -13,8 +13,8 @@ dependencies = [
     # release on a Python 3.12 it sets up once with uv -- `uv` comes as a wheel on 3.8+,
     # 3.7 fetches it with uv's own installer.
     'uv>=0.4; python_version >= "3.8" and python_version < "3.10"',
-    'imperal-mcp>=0.8.12; python_version >= "3.10"',   # raised to the newest at each release
-    'imperal-sdk>=6.3.6; python_version >= "3.10"',
+    'imperal-mcp>=0.9.0; python_version >= "3.10"',   # raised to the newest at each release
+    'imperal-sdk[universal-index]>=6.4.1; python_version >= "3.10"',
     'asyncssh>=2.14; python_version >= "3.10"',
     'httpx>=0.27; python_version >= "3.10"',
     'httpx-sse>=0.4; python_version >= "3.10"',

webbee-0.7.0/src/webbee/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.7.0"

{webbee-0.6.18 → webbee-0.7.0}/src/webbee/intel/service.py

@@ -65,6 +65,9 @@ def _git_ref(root: str) -> str:
 # marker below, which is a fact about the directory rather than a guess about
 # its name.
 _SKIP_DIRS = frozenset({"node_modules", "site-packages", "__pycache__"})
+# macOS packages: an app's or a media library's insides, not the person's files (universal index, 2026-11-04)
+_APP_PACKAGES = (".app", ".photoslibrary", ".musiclibrary", ".tvlibrary", ".photolibrary", ".aplibrary",
+                 ".migratedphotolibrary", ".imovielibrary", ".fcpbundle", ".logicx", ".band")
 
 
 def _is_virtualenv(path: str) -> bool:
@@ -79,7 +82,8 @@ def skipped_dir(parent: str, name: str) -> bool:
     """The ONE rule for what the index never reads (the full walk and the watcher alike):
     every dotted directory (.git, .next build output, .venv, caches, .webbee), the
     dependency caches, and any virtualenv by its marker."""
-    return name.startswith(".") or name in _SKIP_DIRS or _is_virtualenv(os.path.join(parent, name))
+    return (name.startswith(".") or name in _SKIP_DIRS or name.endswith(_APP_PACKAGES)
+            or _is_virtualenv(os.path.join(parent, name)))
 
 
 def ignored_path(root: str, path: str) -> bool:
@@ -129,9 +133,125 @@ def _walk(root: str, limit: int = 20000) -> list[str]:
     return out
 
 
+# ---- universal index (phase 2, 2026-11-04): any folder, every file kind, the focus first ----
+UNIT_MAX_FILES = 100_000       # one index unit (spec 2026-11-03 §3.2); a bigger folder is split by imperal-mcp
+BUILD_BATCH = 2_000            # files per published step of a first build: answers come before the whole folder
+POOL_MIN_FILES = 500           # below this a build stays in-process (a pool's start costs more than it saves)
+RECENT_S = 30 * 86400          # "changed lately" -- indexed right after the focus
+
+
+def _git_listed(root: str) -> set | None:
+    """The files git tracks or would track in ``root`` (``ls-files -co --exclude-standard``: a .gitignore'd file is
+    never read -- the same rule as the cloud listing), or None when ``root`` is no git repository / git cannot say."""
+    if not os.path.exists(os.path.join(root, ".git")):
+        return None
+    try:
+        r = subprocess.run(["git", "-C", root, "ls-files", "-co", "--exclude-standard", "-z"],
+                           capture_output=True, timeout=60)
+    except (OSError, subprocess.SubprocessError):
+        return None
+    if r.returncode != 0:
+        return None
+    return {p.replace(os.sep, "/") for p in r.stdout.decode("utf-8", "surrogateescape").split("\0") if p}
+
+
+def _index_listing(root: str, focus: str | None = None, limit: int = UNIT_MAX_FILES,
+                   exclude: tuple = ()) -> tuple[list, dict, dict]:
+    """``(the folder's files the index reads, in priority order; {relpath: [mtime_ns, size]}; {"placeholder": n,
+    "over_cap": n})`` -- ONE listing for the first build, the cache check and every refresh. Order: the focus folder,
+    files changed in the last 30 days (newest first), then by path. Never read: what ``kind_of`` refuses by the
+    file's name and its real path (secrets, caches, dependency trees), a .gitignore'd file of a git repository,
+    anything but a regular file, and cloud placeholders (reading one downloads it). ``exclude``: root-relative
+    folders another unit indexes (a repository inside a collection), never walked here."""
+    from imperal_sdk.universal.index import is_placeholder
+    from imperal_sdk.universal.kinds import kind_of
+    import stat as _stat
+    found: list[tuple[str, float]] = []
+    stats: dict = {}
+    skipped = {"placeholder": 0, "over_cap": 0}
+    excluded = {str(e).strip("/") for e in exclude if str(e).strip("/")}
+    listed = _git_listed(root)
+    for dp, dns, fns in os.walk(root):
+        here = os.path.relpath(dp, root).replace(os.sep, "/")
+        dns[:] = sorted(d for d in dns if not skipped_dir(dp, d)
+                        and (d if here == "." else f"{here}/{d}") not in excluded)
+        for fn in sorted(fns):
+            rel = fn if here == "." else f"{here}/{fn}"
+            if is_placeholder(rel):
+                skipped["placeholder"] += 1
+                continue
+            if listed is not None and rel not in listed:
+                continue
+            if kind_of(rel, root=root) is None:
+                continue
+            try:
+                st = os.lstat(os.path.join(dp, fn))
+            except OSError:
+                continue
+            if not _stat.S_ISREG(st.st_mode):          # a link, a FIFO, a socket: never read
+                continue
+            if is_placeholder(rel, st):
+                skipped["placeholder"] += 1
+                continue
+            if len(found) >= limit:
+                skipped["over_cap"] += 1
+                continue
+            found.append((rel, st.st_mtime))
+            stats[rel] = [st.st_mtime_ns, st.st_size]
+    prefix = (focus or "").strip("/")
+    now = time.time()
+
+    def rank(item):
+        rel, mtime = item
+        in_focus = bool(prefix) and (rel == prefix or rel.startswith(prefix + "/"))
+        recent = now - mtime <= RECENT_S
+        return (0 if in_focus else 1, 0 if recent else 1, -mtime if recent else 0.0, rel)
+    return [rel for rel, _m in sorted(found, key=rank)], stats, skipped
+
+
+def _index_walk(root: str, focus: str | None = None, limit: int = UNIT_MAX_FILES,
+                exclude: tuple = ()) -> tuple[list[str], dict]:
+    """``(files in priority order, skipped counts)`` -- ``_index_listing`` without the stats."""
+    rels, _stats, skipped = _index_listing(root, focus, limit, exclude)
+    return rels, skipped
+
+
+POOL_MAX_WORKERS = 4           # parsing holds up to ~0.3 GB per job (imperal_sdk.universal.index.RAW_CAP)
+POOL_CHUNK = 50                # files per pool job
+CHUNK_TIMEOUT_S = 120.0        # one job; past it the job's files are retried one by one
+FILE_TIMEOUT_S = 30.0          # one file; past it the file is skipped as ``timeout``
+
+
+def _kill_pool(pool) -> None:
+    """A pool with a stuck job: its processes killed (a hung extractor never ends by itself), the rest cancelled."""
+    for proc in list((getattr(pool, "_processes", None) or {}).values()):
+        try:
+            proc.kill()
+        except Exception:
+            pass
+    try:
+        pool.shutdown(wait=False, cancel_futures=True)
+    except Exception:
+        pass
+
+
+def _build_chunk(root: str, rels: list) -> dict:
+    """One process-pool job: these files' index entries (the pool's children run at a lower priority)."""
+    try:
+        os.nice(10)
+    except (AttributeError, OSError):
+        pass
+    return indexer.build_index(root, rels).files
+
+
 class IntelService:
-    def __init__(self, root: str, repo_key: str, cache_dir: str = "") -> None:
+    def __init__(self, root: str, repo_key: str, cache_dir: str = "", focus: str | None = None,
+                 exclude: tuple = ()) -> None:
         self.root = os.path.realpath(root)
+        self.focus = focus
+        self.exclude = tuple(exclude)
+        self._progress = {"indexed": 0, "total": 0, "skipped": {"placeholder": 0, "over_cap": 0, "unread": 0},
+                          "focus_done": False}
         self.repo_key = repo_key
         self.cache_dir = cache_dir or os.path.expanduser("~/.cache/webbee/intel")
         self.index: ProjectIndex | None = None
@@ -157,6 +277,9 @@ class IntelService:
         self.last_sync_error = ""          # why the last sync failed; "" -- it did not
         self.stale_reason = ""             # why the index is not known fresh; "" -- it is (or not built)
         self._stats: dict = {}             # relpath -> [mtime_ns, size] of what the index holds, saved with it
+        self._tree: dict = {}              # dir -> hash over self._stats (tree.dir_hashes); {} -- compute it again
+        self._checksum = ""                # tree.repo_checksum at the last check ("" -- no git: walk)
+        self._pool_job = None              # the pool job (None: _build_chunk); a test puts a stuck one here
         self._vectors_dirty = False
         self._save_guard = threading.Lock()
         self._save_timer = None
@@ -200,18 +323,34 @@ class IntelService:
         and uncommitted edits stayed invisible after a restart at that HEAD):
         only the files whose mtime or size differ are re-parsed, files gone
         are dropped, new ones added."""
+        touched, rebuilt = self.load_or_build()
+        self._embed_build(self.index, touched)
+        if rebuilt:
+            self._save_cache()             # this index for the next start (a reconciled one too)
+        self.sync_vault()                 # code graph everywhere: the whole repo into the vault
+
+    def load_or_build(self) -> tuple[dict | None, bool]:
+        """The index from the cache checked against the disk, else built: ``(touched, changed)`` -- ``touched`` the
+        files the check re-read or dropped (None after a full build), ``changed`` whether the index differs from the
+        cache (it should be saved). The check compares the checksum trees top down (``tree.changed_files``): a
+        directory whose hash is unchanged is skipped whole. A folder without git is cached too (``store.NO_GIT``)."""
+        from webbee.intel import tree
         self.git_ref = _git_ref(self.root)
-        walked = _walk(self.root)
-        stats = _stats_of(self.root, walked)
-        cached = store.load_with_stats(self.cache_dir, self.repo_key, self.git_ref) if self.git_ref else None
+        checksum = tree.repo_checksum(self.root)
+        walked, stats, skipped = _index_listing(self.root, self.focus, exclude=self.exclude)
+        self._progress = {"indexed": 0, "total": len(walked), "focus_done": False,
+                          "skipped": {**skipped, "unread": 0, "timeout": 0}}
+        cached = store.load_with_stats(self.cache_dir, self.repo_key, self.git_ref)
+        new_tree = tree.dir_hashes(stats)
         touched = None                      # None: a full index (no cache); else the files the disk changed
         if cached is None:
-            idx = indexer.build_index(self.root, walked)
+            idx = self._build_progressive(walked)
         else:
             idx, old_stats = cached
-            changed = [r for r in walked if r in stats and old_stats.get(r) != stats[r]]
-            gone = sorted((set(idx.files) | set(old_stats)) - set(stats))
-            reparsed = indexer.build_index(self.root, changed).files if changed else {}
+            changed, gone, _compared = tree.changed_files(old_stats, stats,
+                                                          store.load_tree(self.cache_dir, self.repo_key), new_tree)
+            gone = sorted(set(gone) | (set(idx.files) - set(stats)))    # a cached file no longer listed
+            reparsed = self._build_files(changed) if changed else {}
             for rel in changed:             # undecodable now: out of the index, as a full build leaves it
                 idx.files.pop(rel, None)
             idx.files.update(reparsed)
@@ -221,14 +360,155 @@ class IntelService:
         idx.git_ref = self.git_ref
         with self._lock:
             self._stats = stats
+            self._tree = new_tree
+            self._checksum = checksum
             self.index = idx
             self.graph = CodeGraph(idx)
+        self._progress.update(indexed=len(walked), focus_done=True)     # every listed file processed (read or unread)
         self.fresh = True
         self.stale_reason = ""
-        self._embed_build(idx, touched)
-        if self.git_ref and (cached is None or touched):
-            self._save_cache()             # this index for the next start (a reconciled one too)
-        self.sync_vault()                 # code graph everywhere: the whole repo into the vault
+        return touched, cached is None or bool(touched)
+
+    def checksum(self) -> str:
+        """The folder's checksum: the root of its checksum tree (``tree[""]``) -- equal checksums, equal listings."""
+        return (self._tree or {}).get("", "")
+
+    def refresh(self) -> dict:
+        """The index catches up with the disk without a watcher: a git repository whose checksum (HEAD, status, the
+        stats of what it names) is unchanged answers at once, with no walk; else one listing, its checksum tree
+        compared top down with the index's, and only the changed files read. A file that left the listing but is
+        still on disk (now .gitignore'd, now named like a secret) leaves the index unread.
+        ``{changed, gone, unlisted, compared}`` or ``{"unchanged": True}``."""
+        from webbee.intel import tree
+        if self.index is None:
+            return {}
+        checksum = tree.repo_checksum(self.root)
+        if checksum and checksum == self._checksum:
+            return {"unchanged": True}
+        walked, stats, _skipped = _index_listing(self.root, self.focus, exclude=self.exclude)
+        new_tree = tree.dir_hashes(stats)
+        with self._lock:
+            old_stats, old_tree = dict(self._stats), dict(self._tree or {})
+        changed, gone, compared = tree.changed_files(old_stats, stats, old_tree, new_tree)
+        with self._lock:
+            gone = sorted(set(gone) | (set(self.index.files) - set(stats)))
+        deleted = [rel for rel in gone if not os.path.lexists(os.path.join(self.root, rel))]
+        unlisted = [rel for rel in gone if rel not in set(deleted)]
+        if unlisted:
+            self.evict(unlisted)
+        if changed or deleted:
+            self.apply_changes(changed + deleted)
+        with self._lock:
+            self._stats = dict(stats)
+            self._tree = new_tree
+            self._checksum = checksum
+        self._progress["total"] = len(walked)
+        self._progress["indexed"] = len(walked)
+        if unlisted and not (changed or deleted):
+            self._save_soon()
+        return {"changed": len(changed), "gone": len(gone), "unlisted": len(unlisted), "compared": compared}
+
+    def evict(self, rels: list) -> None:
+        """Files out of the index without a read (the lock and the graph rebuild apply_changes uses)."""
+        with self._lock:
+            for rel in rels:
+                if self.index is not None:
+                    self.index.files.pop(rel, None)
+                self._stats.pop(rel, None)
+            self._tree = {}
+            if self.index is not None:
+                self.graph = CodeGraph(self.index)
+
+    def _build_files(self, rels: list) -> dict:
+        """These files' index entries. A small set of code stays in-process; documents (any extractor can meet a
+        hostile file) and big sets go to a process pool whose stuck jobs are killed (``_build_in_pool``) -- one hung
+        file never stops the build (final review 2026-11-04)."""
+        if not rels:
+            return {}
+        from imperal_sdk.universal.kinds import kind_of
+        if len(rels) < POOL_MIN_FILES and all(kind_of(r) in ("code", "config") for r in rels):
+            return indexer.build_index(self.root, rels).files
+        try:
+            return self._build_in_pool(rels)
+        except Exception as exc:          # no pool here (a sandbox, a frozen app): the same work in-process
+            log.info("index pool unavailable, building in-process: %s", exc)
+            return indexer.build_index(self.root, rels).files
+
+    def _build_in_pool(self, rels: list) -> dict:
+        """Jobs of ``POOL_CHUNK`` files on at most ``POOL_MAX_WORKERS`` processes; a job past ``CHUNK_TIMEOUT_S`` (or
+        whose process died) has its pool killed and its files retried one per job; a single file past
+        ``FILE_TIMEOUT_S`` is skipped as ``timeout`` (counted in ``progress``)."""
+        workers = max(1, min(POOL_MAX_WORKERS, (os.cpu_count() or 2) // 2))
+        size = POOL_CHUNK if len(rels) >= POOL_CHUNK * workers else max(1, -(-len(rels) // workers))
+        out: dict = {}
+        hung = self._run_pool([rels[i:i + size] for i in range(0, len(rels), size)], workers, CHUNK_TIMEOUT_S, out)
+        if hung:
+            still = self._run_pool([[r] for part in hung for r in part], workers, FILE_TIMEOUT_S, out)
+            for part in still:
+                log.warning("index: %s did not finish in %ss -- skipped", part[0], FILE_TIMEOUT_S)
+                self._progress.setdefault("skipped", {}).setdefault("timeout", 0)
+                self._progress["skipped"]["timeout"] += 1
+        return out
+
+    def _run_pool(self, parts: list, workers: int, timeout: float, out: dict) -> list:
+        """Run ``parts`` in waves of ``workers`` jobs (each job's clock starts with it); the parts that did not end
+        in ``timeout`` or whose process died. A pool that cannot start raises (the caller builds in-process)."""
+        from concurrent.futures import ProcessPoolExecutor, wait
+        hung: list = []
+        pending = list(parts)
+        pool = None
+        try:
+            while pending:
+                if pool is None:
+                    pool = ProcessPoolExecutor(max_workers=workers)
+                wave, pending = pending[:workers], pending[workers:]
+                job = getattr(self, "_pool_job", None) or _build_chunk      # tests put a stuck job here
+                futs = {pool.submit(job, self.root, part): part for part in wave}
+                done, not_done = wait(futs, timeout=timeout)
+                broken = False
+                for f in done:
+                    try:
+                        out.update(f.result())
+                    except Exception:     # the job's process died (a crash, out of memory): its files are retried
+                        hung.append(futs[f])
+                        broken = True
+                hung.extend(futs[f] for f in not_done)
+                if not_done or broken:
+                    _kill_pool(pool)
+                    pool = None
+        finally:
+            if pool is not None:
+                pool.shutdown(wait=True)
+        return hung
+
+    def _build_progressive(self, walked: list) -> ProjectIndex:
+        """A first build in published steps: the focus folder first, then ``BUILD_BATCH`` files at a time -- the index
+        answers (and says how far it is: ``progress``) long before a home folder is done."""
+        idx = ProjectIndex(git_ref=self.git_ref)
+        prefix = (self.focus or "").strip("/")
+        n_focus = sum(1 for r in walked if prefix and (r == prefix or r.startswith(prefix + "/")))
+        steps = ([walked[:n_focus]] if n_focus else []) + [walked[i:i + BUILD_BATCH]
+                                                           for i in range(n_focus, len(walked), BUILD_BATCH)]
+        done = 0
+        for k, step in enumerate(steps):
+            files = self._build_files(step)
+            done += len(step)
+            idx.files.update(files)
+            self._progress["skipped"]["unread"] += len(step) - len(files)
+            self._progress["indexed"] = done            # processed so far; ``skipped.unread`` says how many were not readable
+            if done >= n_focus:
+                self._progress["focus_done"] = True
+            if k == 0 or k % 5 == 0 or k == len(steps) - 1:
+                with self._lock:              # published: search and outline see what is built so far
+                    self.index = idx
+                    self.graph = CodeGraph(idx)
+        return idx
+
+    def progress(self) -> dict:
+        """How far the index is: ``{indexed, total, skipped: {placeholder, over_cap, unread}, focus_done}`` --
+        ``indexed`` counts the listed files processed (``unread`` of them could not be read: encrypted, broken, over
+        their cap); ``indexed == total`` means the build is done."""
+        return {**self._progress, "skipped": dict(self._progress["skipped"])}
 
     def _embed_build(self, idx: ProjectIndex, touched: dict | None = None) -> None:
         # Off-loop already (build() is called via asyncio.to_thread from the
@@ -328,6 +608,7 @@ class IntelService:
             return
         updates = {}
         stats = {}
+        to_read: list = []
         for rel in rel_paths:
             ap = os.path.join(self.root, rel)
             if not os.path.exists(ap):
@@ -336,14 +617,20 @@ class IntelService:
                 continue
             try:
                 st = os.stat(ap)        # before the read: an edit after it differs at the next start
-                with open(ap, "r", encoding="utf-8") as f:
-                    fi = indexer.parse_file(rel, f.read())
-            except UnicodeDecodeError:
-                fi = None               # not text any more: out of the index, as a full build leaves it
             except OSError:
                 continue
-            updates[rel] = fi
+            from imperal_sdk.universal.index import is_placeholder
+            from imperal_sdk.universal.kinds import kind_of
+            if kind_of(rel, root=self.root) is None or is_placeholder(rel, st):
+                updates[rel] = None     # the same gates as the walk: never read, out of the index
+                stats[rel] = None
+                continue
+            to_read.append(rel)
             stats[rel] = [st.st_mtime_ns, st.st_size]
+        # the same reading as a build (every kind through its view, a hung file killed); not readable now: out
+        read = self._build_files(to_read) if to_read else {}
+        for rel in to_read:
+            updates[rel] = read.get(rel)
 
         # Embed compute (chunk + backend inference) stays off-lock, mirroring
         # the parse-then-swap pattern above -- only the vector-store mutation
@@ -373,6 +660,7 @@ class IntelService:
             self.graph = CodeGraph(self.index)
             if embedded is not None:
                 self._swap_vectors(updates, *embedded)
+            self._tree = {}                    # the stats moved: the tree is computed again at the next check
         if updates:
             self._save_soon()
 
@@ -381,8 +669,6 @@ class IntelService:
     def _save_soon(self) -> None:
         """Save the cache ``SAVE_DEBOUNCE_S`` after the first unsaved change -- never once per change (a branch
         switch is one write); not pushed back by later changes, so a session that edits without pause still saves."""
-        if not self.git_ref:
-            return
         with self._save_guard:
             if self._save_timer is not None:
                 return
@@ -407,18 +693,22 @@ class IntelService:
 
     def _save_cache(self) -> None:
         """The index, the stats of its files and (when they changed) the vectors, into the cache. Fail-soft."""
-        if self.index is None or not self.git_ref:
+        if self.index is None:
             return
+        from webbee.intel import tree
         with self._lock:
             snap = ProjectIndex(git_ref=self.git_ref)
             snap.files = dict(self.index.files)
             stats = dict(self._stats)
+            if not self._tree:
+                self._tree = tree.dir_hashes(stats)
+            dirs = dict(self._tree)
             vecs = None
             if self._vectors_dirty and self.vectors is not None:
                 vecs = self.vectors.to_arrays()
                 self._vectors_dirty = False
         try:
-            store.save(self.cache_dir, self.repo_key, snap, stats)
+            store.save(self.cache_dir, self.repo_key, snap, stats, dirs)
             b = self._get_backend() if vecs is not None else None
             if b is not None:
                 store.save_vectors(self.cache_dir, self.repo_key, snap.git_ref, b.model_id, *vecs)
@@ -643,8 +933,11 @@ class IntelService:
         # snapshot -- never re-touch the live dict, which apply_changes (run
         # off-loop by the watcher, F3) may be resizing concurrently.
         with self._lock:
-            files = list((self.index or ProjectIndex()).files.values())
+            every = list((self.index or ProjectIndex()).files.values())
             embedded_chunks = len(self.vectors.ids()) if self.vectors else 0
+            # the profile is the map of the CODE: a summary that leaves this process never names a document's
+            # sections (security review 2026-11-04); other kinds are only counted
+            files = [fi for fi in every if getattr(fi, "kind", "code") in ("code", "config")]
             vectors_ready = self.vectors_ready
             content_hashes = sorted(fi.content_hash for fi in files if fi.content_hash)
             endpoints = [e for fi in files for e in fi.endpoints]
@@ -662,6 +955,8 @@ class IntelService:
             "evidence_version": 1,
             "content_digest": digest,
             "file_count": len(files),
+            "other_kinds": dict(Counter(getattr(fi, "kind", "code") for fi in every
+                                        if getattr(fi, "kind", "code") not in ("code", "config"))),
             "languages": dict(langs),
             "symbol_kinds": dict(kinds),
             "top_symbols": top,               # capped at _MAX_PROFILE_SAMPLE

{webbee-0.6.18 → webbee-0.7.0}/src/webbee/intel/store.py

@@ -23,20 +23,27 @@ from imperal_sdk.code_index import ProjectIndex, FileIndex, Symbol
 # schema 2 holds the OLD id format, so both the index and the vector cache
 # must be treated as a clean miss and rebuilt -- bumping the version is what
 # makes load()/load_vectors() do exactly that.
-SCHEMA_VERSION = 4
+# 5 (2026-11-04): a file's ``kind`` (code | config | document | data | mail | image) is kept -- a schema-4 cache
+# read every document back as code after a restart; a folder without git is cached too (``NO_GIT``) and the
+# checksum tree of the folder (``tree``) is saved with the stats.
+SCHEMA_VERSION = 5
+NO_GIT = "-"           # the cache key of a folder that is no git repository (its stats keep it honest)
 
 
 def _path(cache_dir: str, repo_key: str) -> str:
     return os.path.join(cache_dir, repo_key, "index.json")
 
 
-def save(cache_dir: str, repo_key: str, index: ProjectIndex, stats: dict | None = None) -> None:
+def save(cache_dir: str, repo_key: str, index: ProjectIndex, stats: dict | None = None,
+         tree: dict | None = None) -> None:
     """``stats`` -- ``{relpath: [mtime_ns, size]}`` of every file the walk saw when the index was taken: the next
     start re-parses only the files whose stat differs (edits made while Webbee was closed, uncommitted edits)."""
     p = _path(cache_dir, repo_key)
     os.makedirs(os.path.dirname(p), exist_ok=True)
-    data = {"schema_version": SCHEMA_VERSION, "git_ref": index.git_ref, "stats": stats or {},
-            "files": {k: {"path": v.path, "lang": v.lang, "imports": v.imports, "refs": v.refs,
+    data = {"schema_version": SCHEMA_VERSION, "git_ref": index.git_ref or NO_GIT, "stats": stats or {},
+            "tree": tree or {},
+            "files": {k: {"path": v.path, "lang": v.lang, "kind": getattr(v, "kind", "code"), "imports": v.imports,
+                          "refs": v.refs,
                           "content_hash": v.content_hash, "endpoints": v.endpoints, "schemas": v.schemas,
                           "symbols": [asdict(s) for s in v.symbols]} for k, v in index.files.items()}}
     tmp = p + ".tmp"
@@ -50,6 +57,16 @@ def load(cache_dir: str, repo_key: str, git_ref: str) -> ProjectIndex | None:
     return got[0] if got is not None else None
 
 
+def load_tree(cache_dir: str, repo_key: str) -> dict:
+    """The checksum tree saved with the index (``{dir: hash}``), or ``{}``."""
+    try:
+        with open(_path(cache_dir, repo_key), "r", encoding="utf-8") as f:
+            tree = json.load(f).get("tree") or {}
+        return {str(k): str(v) for k, v in tree.items()} if isinstance(tree, dict) else {}
+    except Exception:
+        return {}
+
+
 def load_with_stats(cache_dir: str, repo_key: str, git_ref: str) -> tuple[ProjectIndex, dict] | None:
     """``(index, {relpath: [mtime_ns, size]})`` saved for this HEAD, or None. A cache from before the stats were kept
     gives ``{}``: every file then counts as changed and is re-parsed once."""
@@ -61,16 +78,17 @@ def load_with_stats(cache_dir: str, repo_key: str, git_ref: str) -> tuple[Projec
     try:
         with open(_path(cache_dir, repo_key), "r", encoding="utf-8") as f:
             data = json.load(f)
-        if data.get("git_ref") != git_ref:
+        if data.get("git_ref") != (git_ref or NO_GIT):
             return None  # stale cache -- the caller falls back to a full re-index
         if data.get("schema_version") != SCHEMA_VERSION:
             return None  # unknown/old on-disk shape -- clean miss, re-index
-        idx = ProjectIndex(git_ref=data.get("git_ref", ""))
+        idx = ProjectIndex(git_ref=git_ref or "")
         for k, v in (data.get("files") or {}).items():
             idx.files[k] = FileIndex(path=v["path"], lang=v["lang"],
                                      imports=v.get("imports", []), refs=v.get("refs", []),
                                      content_hash=v.get("content_hash", ""), endpoints=v.get("endpoints", []),
-                                     schemas=v.get("schemas", []), symbols=[Symbol(**s) for s in v.get("symbols", [])])
+                                     schemas=v.get("schemas", []), symbols=[Symbol(**s) for s in v.get("symbols", [])],
+                                     kind=v.get("kind") or "code")
         stats = {k: list(v) for k, v in (data.get("stats") or {}).items()
                  if isinstance(v, list) and len(v) == 2}
         return idx, stats