imperal-sdk 6.3.7__tar.gz → 6.3.8__tar.gz

{imperal_sdk-6.3.7 → imperal_sdk-6.3.8}/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 All notable changes to `imperal-sdk` are documented here.
 
+## 6.3.8 — 2026-11-04
+
+### Fixed
+- `code_index`: one string-aware reader of resolution configs (`strip_jsonc`, `parse_jsonc`, `parse_config`).
+  Comments and trailing commas are removed outside strings only -- a regex stripper read `"@/*"` in a Next.js
+  `tsconfig.json` as a comment opener, lost `paths`, and every `@/` import fell out of the graph. The Tenant Vault
+  carries a literal copy (a test compares the source).
+- `sftp_list_tree` / `sftp_read_many` keep up to `SFTP_PARALLEL` (32) requests in flight on one connection: listing
+  walks level by level, reads go in slices. Live (Mac -> a US server, 1 572 files): listing 47 s -> 6.7 s, 200
+  files read 206 s -> 10.1 s. Results are the same (paths sorted; .gitignore rules still reach a directory's
+  children before they are filtered).
+
 ## 6.3.7 — 2026-10-21
 
 ### Changed

{imperal_sdk-6.3.7 → imperal_sdk-6.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: imperal-sdk
-Version: 6.3.7
+Version: 6.3.8
 Summary: SDK for building Imperal Cloud extensions
 Author: Imperal, Inc.
 Author-email: Valentin Scerbacov <valentin@imperal.io>

{imperal_sdk-6.3.7 → imperal_sdk-6.3.8}/sdk-reference.json

@@ -1,5 +1,5 @@
 {
-  "sdk_version": "6.3.7",
+  "sdk_version": "6.3.8",
   "symbols": {
     "ActionResult": {
       "action_vocab_safe": false,

{imperal_sdk-6.3.7 → imperal_sdk-6.3.8}/sdk_claims.json

@@ -1,5 +1,5 @@
 {
-  "_sdk_version": "6.3.7",
+  "_sdk_version": "6.3.8",
   "constants": {
     "max_call_depth": {
       "counts_root": true,

{imperal_sdk-6.3.7 → imperal_sdk-6.3.8}/src/imperal_sdk/__init__.py

@@ -1,7 +1,7 @@
 """Imperal Cloud SDK — build extensions for the Imperal platform."""
 from typing import TYPE_CHECKING
 
-__version__ = "6.3.7"
+__version__ = "6.3.8"
 
 # 5.2.2 (2026-06-11): the package root resolves its public surface lazily
 # (PEP 562). The eager imports pulled the HTTP transport (Context / client

{imperal_sdk-6.3.7 → imperal_sdk-6.3.8}/src/imperal_sdk/code_index.py

@@ -746,6 +746,100 @@ def indexable(rel_path: str) -> str | None:
     return "code" if os.path.splitext(low)[1] in INDEXABLE_EXT else None
 
 
+# ---- resolution configs: the ONE reader (the vault's stores/repos.py carries a literal copy) ----
+
+def strip_jsonc(text: str) -> str:
+    """JSON with comments (tsconfig, jsconfig) as plain JSON: ``//`` and ``/* */`` comments and trailing
+    commas removed OUTSIDE strings only. ``"@/*"`` or ``"**/*.ts"`` is text, never a comment (live 2026-11-03:
+    a regex stripper read imperal.io's tsconfig from ``"@/*"`` to ``"**/*.ts"`` as one comment, the JSON
+    broke, ``paths`` was lost and every ``@/`` import fell out of the vault graph)."""
+    n = len(text)
+
+    def skip(i: int) -> int:
+        """The index past whitespace and comments from ``i`` (outside strings)."""
+        while i < n:
+            if text[i] in " \t\r\n":
+                i += 1
+            elif text.startswith("//", i):
+                j = text.find("\n", i)
+                i = n if j < 0 else j
+            elif text.startswith("/*", i):
+                j = text.find("*/", i + 2)
+                i = n if j < 0 else j + 2
+            else:
+                break
+        return i
+
+    out: list = []
+    i = 0
+    while i < n:
+        c = text[i]
+        if c == '"':
+            j = i + 1
+            while j < n and text[j] != '"':
+                j += 2 if text[j] == "\\" else 1
+            out.append(text[i:j + 1])
+            i = j + 1
+        elif text.startswith("//", i) or text.startswith("/*", i):
+            i = skip(i)
+            out.append(" ")
+        elif c == "," and skip(i + 1) < n and text[skip(i + 1)] in "}]":
+            i += 1
+        else:
+            out.append(c)
+            i += 1
+    return "".join(out)
+
+
+def parse_jsonc(text: str) -> dict:
+    """A JSON-with-comments object, or ``{}`` when it is not one."""
+    import json
+    try:
+        data = json.loads(strip_jsonc(text))
+    except ValueError:
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def parse_config(path: str, text: str) -> tuple:
+    """``(kind, data)`` of a resolution config: tsconfig / jsconfig (JSON with comments), pyproject.toml,
+    setup.cfg, package.json. What the linker needs, nothing else."""
+    import posixpath
+    name = posixpath.basename(_slashes(path)).lower()
+    if name == "pyproject.toml":
+        try:
+            import tomllib
+            data = tomllib.loads(text)
+        except Exception:
+            data = {}
+        st = (data.get("tool") or {}).get("setuptools") or {}
+        find = (st.get("packages") or {}).get("find") if isinstance(st.get("packages"), dict) else None
+        return "pyproject", {"where": list((find or {}).get("where") or []),
+                             "package_dir": dict(st.get("package-dir") or {}),
+                             "name": str((data.get("project") or {}).get("name") or "")}
+    if name == "setup.cfg":
+        import configparser
+        cp = configparser.ConfigParser()
+        try:
+            cp.read_string(text)
+            raw = cp.get("options", "package_dir", fallback="")
+        except Exception:
+            raw = ""
+        pdir = {}
+        for line in raw.replace(";", "\n").splitlines():
+            k, sep, v = line.partition("=")
+            if sep:
+                pdir[k.strip()] = v.strip()
+        return "setup_cfg", {"package_dir": pdir}
+    data = parse_jsonc(text)
+    if name == "package.json":
+        return "package_json", {"name": str(data.get("name") or ""), "main": str(data.get("main") or "")}
+    co = data.get("compilerOptions") or {}
+    return ("jsconfig" if name == "jsconfig.json" else "tsconfig"), {
+        "baseUrl": str(co.get("baseUrl") or ""),
+        "paths": {str(k): [str(x) for x in v or []] for k, v in (co.get("paths") or {}).items() if isinstance(v, list)}}
+
+
 # ---- the scope graph of one file (spec 4.6.1): defs, bindings with aliases, reference chains ----
 
 _CHAIN_PARTS = 6
@@ -1872,55 +1966,94 @@ async def sftp_repo_root(sftp, path: str) -> str:
     return manifest or start
 
 
+# SFTP requests in flight on one connection. Live 2026-11-03 (Mac -> whm-jump, /home/ipw-repo): one request at a
+# time listed 1 572 files in 47 s and read 200 files in 206 s (~4 round trips a file) -- a first sync of the kernel
+# repo would have taken half an hour. asyncssh pipelines requests on one channel; 32 keeps a slow link full without
+# flooding a small server's sftp-server.
+SFTP_PARALLEL = 32
+
+
+async def _gather_bounded(coros, limit: int = SFTP_PARALLEL) -> list:
+    import asyncio
+    gate = asyncio.Semaphore(limit)
+
+    async def one(c):
+        async with gate:
+            return await c
+    return await asyncio.gather(*(one(c) for c in coros))
+
+
 async def sftp_list_tree(sftp, root: str, max_files: int = MAX_REPO_FILES) -> dict:
     """Every indexable file under ``root`` with its size and mtime: .gitignore files honoured, dotted /
     dependency / build trees and virtualenvs skipped, symlinks never followed (no loop, nothing outside
-    the root). ``{"items": [{path, size, mtime, kind}], "partial": str, "os"}``."""
+    the root). ``{"items": [{path, size, mtime, kind}], "partial": str, "os"}``. Walked level by level:
+    every directory of a level (and its .gitignore) is read at once; a directory's rules are known
+    before its children are filtered, as in a one-by-one walk."""
     import posixpath
     items: list = []
     partial = ""
     rules: list = []
-    stack = [""]
-    while stack:
-        rel_dir = stack.pop()
+    level = [""]
+
+    async def listed(rel_dir: str):
         here = posixpath.join(root, rel_dir) if rel_dir else root
         try:
-            entries = await sftp.readdir(here)
+            return rel_dir, here, await sftp.readdir(here)
         except Exception:
-            partial = partial or f"unreadable directory: {rel_dir or '.'}"
-            continue
-        names = {e.filename for e in entries}
-        if rel_dir and "pyvenv.cfg" in names:
-            continue                                  # a virtualenv, whatever it is named
-        if ".gitignore" in names:
-            try:
-                data = await _sftp_bytes(sftp, posixpath.join(here, ".gitignore"), 262144)
-                rules += gitignore_rules((data or b"").decode("utf-8", "replace"), rel_dir)
-            except Exception:
-                pass
-        for e in sorted(entries, key=lambda x: x.filename):
-            name = e.filename
-            if name in (".", ".."):
-                continue
-            rel = f"{rel_dir}/{name}" if rel_dir else name
-            kind = getattr(e.attrs, "type", None)
-            if kind == _SFTP_LINK:
+            return rel_dir, here, None
+
+    async def ignore_file(here: str):
+        try:
+            return await _sftp_bytes(sftp, posixpath.join(here, ".gitignore"), 262144)
+        except Exception:
+            return None
+
+    while level and not (partial and partial.startswith("over ")):
+        dirs = sorted(await _gather_bounded([listed(d) for d in level]), key=lambda x: x[0])
+        readable = []
+        for rel_dir, here, entries in dirs:
+            if entries is None:
+                partial = partial or f"unreadable directory: {rel_dir or '.'}"
                 continue
-            if kind == _SFTP_DIR:
-                if name.startswith(".") or name in SKIP_DIRS or name.startswith("Program Files") \
-                        or gitignored(rules, rel, True):
+            names = {e.filename for e in entries}
+            if rel_dir and "pyvenv.cfg" in names:
+                continue                              # a virtualenv, whatever it is named
+            readable.append((rel_dir, here, entries, ".gitignore" in names))
+        ignores = await _gather_bounded([ignore_file(here) for _r, here, _e, has in readable if has])
+        it = iter(ignores)
+        for rel_dir, _here, _entries, has in readable:
+            if has:
+                data = next(it)
+                if data is not None:
+                    rules += gitignore_rules(data.decode("utf-8", "replace"), rel_dir)
+        level = []
+        for rel_dir, _here, entries, _has in readable:
+            for e in sorted(entries, key=lambda x: x.filename):
+                name = e.filename
+                if name in (".", ".."):
                     continue
-                stack.append(rel)
-                continue
-            what = indexable(rel)
-            if not what or gitignored(rules, rel, False):
-                continue
-            if len(items) >= max_files:
-                partial = f"over {max_files} files"
-                stack = []
+                rel = f"{rel_dir}/{name}" if rel_dir else name
+                kind = getattr(e.attrs, "type", None)
+                if kind == _SFTP_LINK:
+                    continue
+                if kind == _SFTP_DIR:
+                    if name.startswith(".") or name in SKIP_DIRS or name.startswith("Program Files") \
+                            or gitignored(rules, rel, True):
+                        continue
+                    level.append(rel)
+                    continue
+                what = indexable(rel)
+                if not what or gitignored(rules, rel, False):
+                    continue
+                if len(items) >= max_files:
+                    partial = f"over {max_files} files"
+                    level = []
+                    break
+                items.append({"path": rel, "size": int(getattr(e.attrs, "size", 0) or 0),
+                              "mtime": float(getattr(e.attrs, "mtime", 0) or 0), "kind": what})
+            if partial.startswith("over "):
                 break
-            items.append({"path": rel, "size": int(getattr(e.attrs, "size", 0) or 0),
-                          "mtime": float(getattr(e.attrs, "mtime", 0) or 0), "kind": what})
+    items.sort(key=lambda i: i["path"])
     return {"items": items, "partial": partial, "os": sftp_os(root)}
 
 
@@ -1933,27 +2066,34 @@ async def sftp_read_many(sftp, root: str, paths: list, max_bytes: int = MAX_FILE
     skipped: list = []
     rest: list = []
     used = 0
+    todo: list = []
     for rel in paths or []:
         full = _inside(root, str(rel))
         if full is None or not indexable(str(rel)):
             skipped.append(rel)
-            continue
-        if used >= batch_bytes:
-            rest.append(rel)
-            continue
+        else:
+            todo.append((rel, full))
+
+    async def one(rel, full):
         if not await _sftp_contained(sftp, root, full):
-            skipped.append(rel)
-            continue
+            return rel, None
         try:
-            data = await _sftp_bytes(sftp, full, max_bytes)
+            return rel, await _sftp_bytes(sftp, full, max_bytes)
         except Exception:
-            skipped.append(rel)
-            continue
-        if data is None:
-            skipped.append(rel)
-            continue
-        files[rel] = data
-        used += len(data)
+            return rel, None
+
+    # read SFTP_PARALLEL files at once, a slice at a time: the batch cap is checked between slices (a batch may
+    # end one slice over it, as a one-by-one read could end one file over it)
+    for i in range(0, len(todo), SFTP_PARALLEL):
+        if used >= batch_bytes:
+            rest += [rel for rel, _full in todo[i:]]
+            break
+        for rel, data in await _gather_bounded([one(rel, full) for rel, full in todo[i:i + SFTP_PARALLEL]]):
+            if data is None:
+                skipped.append(rel)
+            else:
+                files[rel] = data
+                used += len(data)
     return {"files": files, "skipped": skipped, "rest": rest, "os": sftp_os(root)}
 
 

imperal_sdk-6.3.8/tests/test_code_index_configs.py

@@ -0,0 +1,70 @@
+"""The one reader of resolution configs (tsconfig/jsconfig are JSON with comments): comments and trailing
+commas go, strings stay whole -- live 2026-11-03, imperal.io's ``"@/*"`` was read as a comment opener."""
+from imperal_sdk.code_index import parse_config, parse_jsonc, strip_jsonc
+
+# imperal.io's own tsconfig.json (Next.js 16), verbatim in shape: "@/*" and "**/*.ts" in strings
+NEXT_TSCONFIG = """{
+  "compilerOptions": {
+    "target": "ES2017",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "strict": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./src/*"]
+    }
+  },
+  "include": [
+    "next-env.d.ts",
+    "**/*.ts",
+    "**/*.tsx",
+    ".next/types/**/*.ts"
+  ],
+  "exclude": ["node_modules"]
+}
+"""
+
+
+def test_next_tsconfig_keeps_its_paths():
+    kind, data = parse_config("tsconfig.json", NEXT_TSCONFIG)
+    assert kind == "tsconfig"
+    assert data == {"baseUrl": "", "paths": {"@/*": ["./src/*"]}}
+
+
+def test_comments_and_trailing_commas_go_outside_strings_only():
+    text = """{
+      // a line comment with "quotes" and /* inside
+      "compilerOptions": {
+        /* a block comment
+           over lines, with "@/*" text */
+        "baseUrl": ".", // after a value
+        "paths": {"~/*": ["src/*",], "url": ["https://example.com/a//b"],},
+      },
+    }"""
+    data = parse_jsonc(text)
+    assert data["compilerOptions"]["baseUrl"] == "."
+    assert data["compilerOptions"]["paths"] == {"~/*": ["src/*"], "url": ["https://example.com/a//b"]}
+
+
+def test_escaped_quotes_and_backslashes_stay_inside_the_string():
+    text = '{"a": "say \\"/*\\" not a comment", "b": "c:\\\\dir\\\\", "c": 1,}'
+    assert parse_jsonc(text) == {"a": 'say "/*" not a comment', "b": "c:\\dir\\", "c": 1}
+
+
+def test_a_comma_before_a_comment_then_a_brace_is_trailing():
+    assert parse_jsonc('{"a": [1, 2, /* end */ ], "b": 3, // last\n}') == {"a": [1, 2], "b": 3}
+
+
+def test_not_an_object_is_empty():
+    assert parse_jsonc("[1, 2]") == {}
+    assert parse_jsonc("{ broken") == {}
+    assert strip_jsonc("") == ""
+
+
+def test_package_json_and_jsconfig():
+    assert parse_config("web/package.json", '{"name": "web", "main": "index.js", // c\n}') == (
+        "package_json", {"name": "web", "main": "index.js"})
+    assert parse_config("jsconfig.json", '{"compilerOptions": {"baseUrl": "src"}}')[0] == "jsconfig"

{imperal_sdk-6.3.7 → imperal_sdk-6.3.8}/tests/test_code_index_indexer.py

@@ -468,3 +468,63 @@ def test_a_source_with_an_embedder_says_so_to_the_vault_and_one_without_never_do
     monkeypatch.setattr(ci, "_backend", lambda: None)
     asyncio.run(ci.sync_source(plan, apply, {"source": "t"}, listing, read))
     assert [m.get("embeds") for m in seen] == [True, None, None]
+
+
+def test_sftp_listing_and_reads_run_in_parallel_with_the_same_result():
+    """live 2026-11-03 (Mac -> whm-jump): one SFTP request at a time listed 1 572 files in 47 s and read 200 in 206 s;
+    in flight together the same listing took 6.7 s and the reads 10.1 s. The result is the walk's, rules included."""
+    import asyncio
+    import time
+    from types import SimpleNamespace as NS
+
+    from imperal_sdk import code_index as ci
+
+    def ent(name, kind, size=10):
+        return NS(filename=name, attrs=NS(type=kind, size=size, mtime=1.0))
+    tree = {"/r": [ent(".gitignore", 1), ent("pkg", ci._SFTP_DIR), ent("gen", ci._SFTP_DIR), ent("top.py", 1)],
+            "/r/pkg": [ent(f"m{i}.py", 1) for i in range(60)] + [ent("sub", ci._SFTP_DIR), ent(".gitignore", 1)],
+            "/r/pkg/sub": [ent("keep.py", 1), ent("skip_me.py", 1)],
+            "/r/gen": [ent("out.py", 1)]}
+    files = {"/r/.gitignore": b"gen/\n", "/r/pkg/.gitignore": b"sub/skip_me.py\n",
+             **{f"/r/pkg/m{i}.py": b"x = 1\n" for i in range(60)}, "/r/top.py": b"y = 2\n"}
+    live = {"now": 0, "peak": 0}
+
+    class F:
+        def __init__(self, data):
+            self.data = data
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *a):
+            return False
+
+        async def read(self, n=-1):
+            live["now"] += 1
+            live["peak"] = max(live["peak"], live["now"])
+            await asyncio.sleep(0.02)
+            live["now"] -= 1
+            return self.data
+
+    class S:
+        async def readdir(self, p):
+            await asyncio.sleep(0.02)
+            return tree[p]
+
+        async def realpath(self, p):
+            return p
+
+        def open(self, p, mode="rb"):
+            if p not in files:
+                raise OSError(p)
+            return F(files[p])
+    got = asyncio.run(ci.sftp_list_tree(S(), "/r"))
+    paths = [i["path"] for i in got["items"]]
+    assert paths == sorted(paths) and "top.py" in paths and "pkg/sub/keep.py" in paths and len(paths) == 62
+    assert "gen/out.py" not in paths and "pkg/sub/skip_me.py" not in paths       # parent rules reach the children
+    t0 = time.monotonic()
+    many = asyncio.run(ci.sftp_read_many(S(), "/r", [p for p in paths if p.startswith("pkg/m")]))
+    assert len(many["files"]) == 60 and not many["skipped"] and not many["rest"]
+    assert live["peak"] > 1 and time.monotonic() - t0 < 60 * 0.02            # overlapped, not one by one
+    capped = asyncio.run(ci.sftp_read_many(S(), "/r", [p for p in paths if p.startswith("pkg/m")], batch_bytes=10))
+    assert capped["files"] and capped["rest"] and len(capped["files"]) + len(capped["rest"]) == 60