imperal-sdk 6.3.2__tar.gz → 6.3.4__tar.gz

{imperal_sdk-6.3.2 → imperal_sdk-6.3.4}/CHANGELOG.md

@@ -2,6 +2,20 @@
 
 All notable changes to `imperal-sdk` are documented here.
 
+## 6.3.4 — 2026-10-20
+
+### Changed
+- Cloud OS limits for the code index: a file up to 10 MB (`MAX_FILE_BYTES`); an apply part up to 10 MB as sent
+  and at most 20 file versions (`APPLY_BATCH_FILES`); a file bigger than one part travels alone.
+- A huge generated file whose facts would not fit one apply is stored as text only and named in
+  `partial_reason` -- a sync never fails on it.
+
+## 6.3.3 — 2026-10-20
+
+### Fixed
+- `apply_batches` sizes each part as it is sent (the blob's text and facts, its chunks' vectors) against
+  `APPLY_BATCH_BYTES` = 4 MB, half the vault's request cap: a first sync of a large repo no longer gets 413.
+
 ## 6.3.2 — 2026-10-20
 
 ### Security

{imperal_sdk-6.3.2 → imperal_sdk-6.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: imperal-sdk
-Version: 6.3.2
+Version: 6.3.4
 Summary: SDK for building Imperal Cloud extensions
 Author: Valentin Scerbacov, Imperal, Inc.
 License-Expression: Apache-2.0

{imperal_sdk-6.3.2 → imperal_sdk-6.3.4}/sdk-reference.json

@@ -1,5 +1,5 @@
 {
-  "sdk_version": "6.3.2",
+  "sdk_version": "6.3.4",
   "symbols": {
     "ActionResult": {
       "action_vocab_safe": false,

{imperal_sdk-6.3.2 → imperal_sdk-6.3.4}/sdk_claims.json

@@ -1,5 +1,5 @@
 {
-  "_sdk_version": "6.3.2",
+  "_sdk_version": "6.3.4",
   "constants": {
     "max_call_depth": {
       "counts_root": true,

{imperal_sdk-6.3.2 → imperal_sdk-6.3.4}/src/imperal_sdk/__init__.py

@@ -1,7 +1,7 @@
 """Imperal Cloud SDK — build extensions for the Imperal platform."""
 from typing import TYPE_CHECKING
 
-__version__ = "6.3.2"
+__version__ = "6.3.4"
 
 # 5.2.2 (2026-06-11): the package root resolves its public surface lazily
 # (PEP 562). The eager imports pulled the HTTP transport (Context / client

{imperal_sdk-6.3.2 → imperal_sdk-6.3.4}/src/imperal_sdk/code_index.py

@@ -642,7 +642,7 @@ def _lang_for(path: str) -> str:
 # ---- what any source may index (spec 4.2: one list, enforced by every source, refused by the vault) ----
 
 EXTRACTOR_VERSION = "2"
-MAX_FILE_BYTES = 1_048_576
+MAX_FILE_BYTES = 10 * 1024 * 1024     # one file in the code index (Cloud OS: shared vault, many users)
 MAX_REPO_FILES = 20_000
 INDEXABLE_EXT = (".py", ".pyi", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs")
 # the resolution configs (read and stored parsed); setup.py is code to execute -- never read
@@ -1610,20 +1610,33 @@ def embed_query(text: str) -> bytes:
 # normalised and hashed, only blobs the repo does not hold are extracted (and their chunks embedded),
 # and the apply goes in size-bounded parts -- the last one stitches the links.
 
-APPLY_BATCH_BYTES = 8_000_000
+APPLY_BATCH_BYTES = 10_000_000     # one apply body as sent (JSON); the vault and nginx take 32 MiB on sync/apply --
+#                                    a part holds one file alone when that file is bigger (up to MAX_FILE_BYTES as text)
+APPLY_BATCH_FILES = 20             # and at most this many new file versions per apply
 READ_BATCH = 200
 
 
-def apply_batches(files: list, blobs: list, chunks: list, limit: int = APPLY_BATCH_BYTES) -> list:
+def _sent_size(blob: dict, chunks) -> int:
+    """A blob's size as an apply sends it: the blob with its text and facts, and its chunks with their vectors."""
+    import json as _json
+    return len(_json.dumps(blob, default=str).encode("utf-8")) + sum(
+        len(_json.dumps(c, default=str).encode("utf-8")) for c in chunks or ())
+
+
+def apply_batches(files: list, blobs: list, chunks: list, limit: int = APPLY_BATCH_BYTES,
+                  max_blobs: int = APPLY_BATCH_FILES) -> list:
     """One sync's apply split by size (a big first sync never goes as one HTTP body): each part carries
     its blobs, the files that point at them and their chunks; files of blobs the repo already holds go
     first. ``[{files, blobs, chunks}]`` -- the last part stitches."""
     parts: list = [{"files": [], "blobs": [], "chunks": []}]
     where: dict = {}
     size = 0
+    by_blob: dict = {}
+    for c in chunks:
+        by_blob.setdefault(c.get("blob"), []).append(c)
     for b in blobs:
-        n = len(str(b.get("text") or "").encode("utf-8"))
-        if parts[-1]["blobs"] and size + n > limit:
+        n = _sent_size(b, by_blob.get(b["hash"], ()))
+        if parts[-1]["blobs"] and (size + n > limit or len(parts[-1]["blobs"]) >= max_blobs):
             parts.append({"files": [], "blobs": [], "chunks": []})
             size = 0
         parts[-1]["blobs"].append(b)
@@ -1667,7 +1680,7 @@ async def sync_source(plan, apply, meta: dict, listing: list, read, *, embed_chu
         if not p or not indexable(p):
             continue
         if int(it.get("size") or 0) > MAX_FILE_BYTES:
-            partial.append(f"{p} over 1 MB")
+            partial.append(f"{p} over {MAX_FILE_BYTES // 1048576} MB")
             continue
         items.append({"path": p, "size": int(it.get("size") or 0), "mtime": float(it.get("mtime") or 0.0)})
     if len(items) > MAX_REPO_FILES:
@@ -1702,7 +1715,7 @@ async def sync_source(plan, apply, meta: dict, listing: list, read, *, embed_chu
             data = raw if isinstance(raw, (bytes, bytearray)) else str(raw or "").encode("utf-8")
             read_bytes += len(data)
             if len(data) > MAX_FILE_BYTES:
-                partial.append(f"{p} over 1 MB")
+                partial.append(f"{p} over {MAX_FILE_BYTES // 1048576} MB")
                 continue
             text = normalise_text(bytes(data))
             if text is None:
@@ -1719,9 +1732,14 @@ async def sync_source(plan, apply, meta: dict, listing: list, read, *, embed_chu
             facts = scope_facts(p, text)
             body = facts.to_dict() if facts is not None else {"lang": "", "lines": text.count("\n") + 1}
             extract_ms += (_time.monotonic() - e0) * 1000
+            mine = chunks_for(h, facts, text) if embed_chunks else []
+            if _sent_size({"hash": h, "text": text, "facts": body}, mine) > APPLY_BATCH_BYTES:
+                # a huge generated file (a bundle, a dump): its facts would not fit one apply -- kept as text,
+                # never parsed into the graph, and said
+                body, mine = {"lang": body.get("lang") or "", "lines": text.count("\n") + 1}, []
+                partial.append(f"{p} kept as text only (too large to parse)")
             blobs.append({"hash": h, "lang": body.get("lang") or "", "text": text, "facts": body})
-            if embed_chunks:
-                chunks += chunks_for(h, facts, text)
+            chunks += mine
     meta["partial_reason"] = ("; ".join(dict.fromkeys(partial)))[:1000] or None
     removed = list(planned.get("removed") or [])
     counts: dict = {}

{imperal_sdk-6.3.2 → imperal_sdk-6.3.4}/tests/test_code_index_indexer.py

@@ -95,7 +95,7 @@ def test_one_table_and_the_old_private_names_stay_aliases():
     assert indexer.DEF_NODES["typescript"]["type_alias_declaration"] == "type"
     assert indexer.DEF_NODES["typescript"]["enum_declaration"] == "class"
     assert indexer._DEF_NODES is indexer.DEF_NODES and indexer._get_language is indexer.language
-    assert indexer.EXTRACTOR_VERSION == "2" and indexer.MAX_FILE_BYTES == 1_048_576
+    assert indexer.EXTRACTOR_VERSION == "2" and indexer.MAX_FILE_BYTES == 10 * 1024 * 1024
     assert indexer.MAX_REPO_FILES == 20_000
 
 
@@ -252,7 +252,7 @@ def test_sync_source_reads_only_changed_and_batches_the_apply(tmp_path):
     assert ".env" not in [i["path"] for i in plans[0][0]] and "same.py" not in reads[0]
     assert len(applies) == 1 and len(applies[0]["blobs"]) == 1 and len(applies[0]["files"]) == 2   # one blob, two files
     assert applies[0]["removed"] == ["gone.py"] and applies[0]["configs"][0]["path"] == "pyproject.toml"
-    assert "lost.py" in out["partial_reason"] and "big.py over 1 MB" in out["partial_reason"]
+    assert "lost.py" in out["partial_reason"] and "big.py over 10 MB" in out["partial_reason"]
     parts = ci.apply_batches([{"path": "a", "blob": "h1"}, {"path": "b", "blob": "h2"}],
                              [{"hash": "h1", "text": "x" * 10}, {"hash": "h2", "text": "y" * 10}], [], limit=15)
     assert [len(p["blobs"]) for p in parts] == [1, 1] and parts[1]["files"][0]["path"] == "b"
@@ -393,3 +393,53 @@ def test_a_remote_never_carries_credentials_however_it_is_written():
     assert clean_remote("bob:secret@host:r.git") == "host:r.git"
     assert clean_remote("https://[::1]:9/r") == "https://[::1]:9/r" and clean_remote("http://h:bad/r") == ""
     assert clean_remote("") == "" and clean_remote("a b") == ""
+
+
+def test_every_apply_part_fits_the_vault_body_cap_as_sent():
+    """Live 2026-10-20: a first sync of the kernel repo hit the vault's 8 MiB body cap (413) -- the parts
+    were sized by file text alone while each blob also carries its facts and its chunks' vectors."""
+    import json
+    from imperal_sdk import code_index as ci
+    blobs = [{"hash": f"h{i}", "text": "x" * 200_000,
+              "facts": {"defs": [{"name": f"d{j}", "qualname": f"q{j}" * 20} for j in range(4000)]}} for i in range(40)]
+    files = [{"path": f"f{i}.py", "blob": f"h{i}"} for i in range(40)]
+    chunks = [{"blob": f"h{i}", "vector": "A" * 2000, "start_line": k} for i in range(40) for k in range(50)]
+    parts = ci.apply_batches(files, blobs, chunks)
+    assert len(parts) > 1 and sum(len(p["blobs"]) for p in parts) == 40
+    for p in parts:
+        assert len(json.dumps(p).encode("utf-8")) <= ci.APPLY_BATCH_BYTES + 64_000
+        assert len(p["blobs"]) <= ci.APPLY_BATCH_FILES == 20
+    assert ci.APPLY_BATCH_BYTES <= 10_000_000 < 16 * 1024 * 1024          # under the vault / nginx sync bound
+    small = ci.apply_batches([{"path": f"s{i}", "blob": f"s{i}"} for i in range(45)],
+                             [{"hash": f"s{i}", "text": "x"} for i in range(45)], [])
+    assert [len(p["blobs"]) for p in small] == [20, 20, 5]
+
+
+def test_a_file_whose_facts_do_not_fit_an_apply_is_kept_as_text_and_said(monkeypatch):
+    """Cloud OS limits: a file up to 10 MB is indexed; a huge generated one whose facts would not fit one
+    apply is stored as text only -- never a failed sync."""
+    import asyncio
+    from imperal_sdk import code_index as ci
+    assert ci.MAX_FILE_BYTES == 10 * 1024 * 1024
+    monkeypatch.setattr(ci, "APPLY_BATCH_BYTES", 20_000)
+    applies = []
+
+    async def plan(items, meta):
+        return {"changed": [i["path"] for i in items], "removed": [], "have_blobs": []}
+
+    async def apply(**payload):
+        applies.append(payload)
+        return {"files": len(payload["files"])}
+
+    big = "".join(f"def f{i}(a):\n    return g{i}(a)\n" for i in range(800))
+    files = {"small.py": "def a():\n    return 1\n", "bundle.py": big}
+
+    async def read(paths):
+        return {p: files[p].encode() for p in paths}
+    out = asyncio.run(ci.sync_source(plan, apply, {"source": "t"}, [{"path": p, "size": len(t), "mtime": 1.0}
+                                                                    for p, t in files.items()], read, embed_chunks=False))
+    sent = {b["hash"]: b for a in applies for b in a["blobs"]}
+    bundle = next(b for b in sent.values() if b["text"] == big)
+    assert "defs" not in bundle["facts"] and bundle["facts"]["lines"] > 1000
+    assert any(b["facts"].get("defs") for b in sent.values() if b["text"] != big)
+    assert "bundle.py kept as text only" in out["partial_reason"]