webbee 0.3.52__tar.gz → 0.3.53__tar.gz

{webbee-0.3.52 → webbee-0.3.53}/CHANGELOG.md

@@ -1,5 +1,46 @@
 # Changelog
 
+## 0.3.53
+
+**Documents and images are now readable — natively, on every OS.**
+
+`read_file` opened every path as UTF-8. A `.docx`, `.pdf`, `.xlsx` or a
+screenshot therefore did not read *badly* — it raised `UnicodeDecodeError` and
+the agent got **nothing at all**, on macOS, Windows and Linux alike. The
+workaround was a detour through `bash` (`strings`, a converter, an OCR tool)
+that burned a turn and produced garbage.
+
+Meanwhile the platform already reads those formats: the document service on
+the API server, reached through the **system File Reader extension**, handles
+pdf/docx/doc/odt/rtf/xlsx/ods/csv/pptx/odp and images (OCR + vision). The
+capability existed; the bridge from a *local* path to it did not.
+
+That bridge is what this release adds — and nothing else:
+
+* **No parsers on the client.** Extraction stays the service's job, reached
+  only through the file-reader extension. Vendoring `pdfplumber` here would
+  have created a second, drifting source of truth.
+* **Bytes never enter the model's context.** They ride base64 in the extension
+  call — the same door the panel dropzone, Telegram and clipboard paste
+  already use — and the brain gets back a compact **text window** plus a
+  `file_id` it pages through with `offset`/`limit`. A 300-page PDF costs a
+  window, not a fortune in tokens.
+* **Source files are untouched.** They keep the byte-exact text path that
+  `edit_file`'s exact-match contract depends on. `.svg` stays text: it is
+  markup an agent edits, not a picture to OCR.
+* **Honest degradation.** Offline, no plan, service down, still extracting —
+  each is one readable sentence, never a traceback, never a silently empty
+  read.
+* **A binary that hides its type** (no extension, an odd one, a mislabelled
+  export) is caught on the decode error and routed the same way, instead of
+  failing with an error the agent can do nothing about.
+
+Two bugs found by reading real files end-to-end rather than trusting the
+schema: the extracted text arrives as `body` (reading only `text` returned a
+confident, completely empty answer), and a not-yet-extracted file answers with
+a human placeholder sentence — treating that as content made the wait exit on
+its first tick and declare a perfectly readable image empty.
+
 ## 0.3.52
 
 Tool-honesty pass. An audit of the local tools against live behaviour found

{webbee-0.3.52 → webbee-0.3.53}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: webbee
-Version: 0.3.52
+Version: 0.3.53
 Summary: Webbee 🐝 — the Imperal Cloud coding agent in your terminal
 Project-URL: Homepage, https://imperal.io
 Project-URL: Documentation, https://docs.imperal.io

{webbee-0.3.52 → webbee-0.3.53}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "webbee"
-version = "0.3.52"
+version = "0.3.53"
 description = "Webbee 🐝 — the Imperal Cloud coding agent in your terminal"
 readme = "README.md"
 license = "MIT"

webbee-0.3.53/src/webbee/intel/contracts.py

@@ -0,0 +1,183 @@
+"""Deterministic endpoint/schema evidence extraction.
+
+This is deliberately a projection over source snapshots, not a second source
+of truth. Every record carries file+line provenance and is rebuilt from the
+content-addressed repository index. Unsupported syntax is simply absent; no
+LLM inference is stored as fact.
+"""
+from __future__ import annotations
+
+import ast
+import re
+from dataclasses import asdict, dataclass
+from typing import Iterable
+
+_HTTP_METHODS = frozenset({"get", "post", "put", "patch", "delete", "options", "head"})
+
+
+@dataclass(frozen=True)
+class EndpointEvidence:
+    method: str
+    route: str
+    handler: str
+    path: str
+    line: int
+    schema_refs: tuple[str, ...] = ()
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class SchemaEvidence:
+    name: str
+    schema_kind: str
+    path: str
+    line: int
+    fields: tuple[str, ...] = ()
+    storage_name: str = ""
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+def _name(node) -> str:
+    if isinstance(node, ast.Name):
+        return node.id
+    if isinstance(node, ast.Attribute):
+        left = _name(node.value)
+        return f"{left}.{node.attr}" if left else node.attr
+    if isinstance(node, ast.Subscript):
+        return _name(node.value)
+    return ""
+
+
+def _literal_string(node) -> str:
+    return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else ""
+
+
+def _annotation_names(node) -> set[str]:
+    if node is None:
+        return set()
+    names = set()
+    for part in ast.walk(node):
+        n = _name(part)
+        if n and n.split(".")[-1][:1].isupper():
+            names.add(n.split(".")[-1])
+    return names
+
+
+def _python_contracts(path: str, text: str):
+    try:
+        tree = ast.parse(text)
+    except (SyntaxError, ValueError):
+        return [], []
+    endpoints, schemas = [], []
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            for dec in node.decorator_list:
+                if not isinstance(dec, ast.Call) or not isinstance(dec.func, ast.Attribute):
+                    continue
+                method = dec.func.attr.lower()
+                route = _literal_string(dec.args[0]) if dec.args else ""
+                if method not in _HTTP_METHODS or not route.startswith("/"):
+                    continue
+                refs = set()
+                for arg in (*node.args.posonlyargs, *node.args.args, *node.args.kwonlyargs):
+                    refs.update(_annotation_names(arg.annotation))
+                refs.update(_annotation_names(node.returns))
+                for kw in dec.keywords:
+                    if kw.arg in {"response_model", "request_model", "model"}:
+                        refs.update(_annotation_names(kw.value))
+                endpoints.append(EndpointEvidence(
+                    method.upper(), route, node.name, path,
+                    getattr(dec, "lineno", node.lineno), tuple(sorted(refs))))
+        elif isinstance(node, ast.ClassDef):
+            bases = {_name(b).split(".")[-1] for b in node.bases}
+            is_model = bool(bases & {"BaseModel", "TypedDict", "Schema", "Serializer"})
+            is_table = bool(bases & {"Base", "DeclarativeBase", "Model"})
+            if not (is_model or is_table):
+                continue
+            fields, storage = set(), ""
+            for stmt in node.body:
+                if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name):
+                    fields.add(stmt.target.id)
+                elif isinstance(stmt, ast.Assign):
+                    for target in stmt.targets:
+                        if isinstance(target, ast.Name):
+                            if target.id == "__tablename__":
+                                storage = _literal_string(stmt.value)
+                            elif isinstance(stmt.value, ast.Call):
+                                call = _name(stmt.value.func).split(".")[-1]
+                                if call in {"Column", "mapped_column", "Field"}:
+                                    fields.add(target.id)
+            schemas.append(SchemaEvidence(
+                node.name, "table" if is_table else "model", path, node.lineno,
+                tuple(sorted(fields)), storage))
+    return endpoints, schemas
+
+
+_TS_ENDPOINT = re.compile(
+    r"(?m)\b(?:app|router|server)\s*\.\s*(get|post|put|patch|delete|options|head)"
+    r"\s*\(\s*([\"'])(/[^\"']*)\2\s*,\s*([A-Za-z_$][\w$]*)"
+)
+_TS_SCHEMA = re.compile(
+    r"(?ms)\b(?:export\s+)?(?:interface|type|class)\s+([A-Za-z_$][\w$]*)"
+    r"(?:\s+extends[^\{=]+)?\s*(?:\{|=\s*\{)(.*?)\}"
+)
+_TS_FIELD = re.compile(r"(?m)^\s*([A-Za-z_$][\w$]*)\s*\??\s*:")
+
+
+def _typescript_contracts(path: str, text: str):
+    endpoints = [EndpointEvidence(
+        m.group(1).upper(), m.group(3), m.group(4), path,
+        text.count("\n", 0, m.start()) + 1)
+        for m in _TS_ENDPOINT.finditer(text)]
+    schemas = []
+    for m in _TS_SCHEMA.finditer(text):
+        schemas.append(SchemaEvidence(
+            m.group(1), "model", path, text.count("\n", 0, m.start()) + 1,
+            tuple(sorted(set(_TS_FIELD.findall(m.group(2)))))))
+    return endpoints, schemas
+
+
+def extract_contracts(path: str, text: str, lang: str):
+    if lang == "python":
+        return _python_contracts(path, text)
+    if lang in {"typescript", "javascript"}:
+        return _typescript_contracts(path, text)
+    return [], []
+
+
+def match_contracts(repos: Iterable[dict]) -> list[dict]:
+    """Build deterministic cross-repo links from exact contract identities.
+
+    Endpoint route+method and schema names are intentionally exact. Semantic
+    guesses belong in retrieval ranking, never in the evidence graph.
+    """
+    repos = sorted(repos, key=lambda r: r.get("repo_key", ""))
+    out = []
+    for i, left in enumerate(repos):
+        for right in repos[i + 1:]:
+            lk, rk = left.get("repo_key", ""), right.get("repo_key", "")
+            le = {(x.method, x.route): x for x in left.get("endpoints", ())}
+            re_ = {(x.method, x.route): x for x in right.get("endpoints", ())}
+            for key in sorted(le.keys() & re_.keys()):
+                out.append({"relation": "endpoint_identity", "source_repo": lk,
+                            "target_repo": rk, "identity": f"{key[0]} {key[1]}"})
+            # A schema contract can be declared as a concrete model/table OR
+            # referenced by an endpoint annotation/response_model. Both are
+            # deterministic source evidence; including references lets a
+            # producer endpoint link to a consumer's shared DTO even when the
+            # producer imports that DTO from another package.
+            def schema_names(repo):
+                names = {x.name for x in repo.get("schemas", ())}
+                for endpoint in repo.get("endpoints", ()):
+                    names.update(endpoint.schema_refs)
+                return names
+
+            for name in sorted(schema_names(left) & schema_names(right)):
+                source, target = sorted((lk, rk))
+                out.append({"relation": "schema_name_match", "source_repo": source,
+                            "target_repo": target, "identity": name})
+    return out

{webbee-0.3.52 → webbee-0.3.53}/src/webbee/intel/indexer.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
+import hashlib
 import os
 
+from webbee.intel.contracts import extract_contracts
 from webbee.intel.models import FileIndex, ProjectIndex, Symbol
 
 LANG_BY_EXT = {".py": "python", ".ts": "typescript", ".tsx": "typescript",
@@ -59,7 +61,10 @@ def _name_of(node, src_bytes: bytes) -> str:
 
 def parse_file(path: str, text: str) -> FileIndex | None:
     lang = _lang_for(path)
-    fi = FileIndex(path=path, lang=lang)
+    fi = FileIndex(path=path, lang=lang, content_hash=hashlib.sha256(text.encode("utf-8", "replace")).hexdigest())
+    endpoints, schemas = extract_contracts(path, text, lang)
+    fi.endpoints = [x.to_dict() for x in endpoints]
+    fi.schemas = [x.to_dict() for x in schemas]
     if lang == "other":
         return fi  # line-only fallback (no symbols) — never crash
     L = _get_language(lang)

{webbee-0.3.52 → webbee-0.3.53}/src/webbee/intel/models.py

@@ -19,6 +19,9 @@ class FileIndex:
     symbols: list[Symbol] = field(default_factory=list)
     imports: list[str] = field(default_factory=list)   # imported module/path strings
     refs: list[str] = field(default_factory=list)      # referenced identifier names (for the graph)
+    content_hash: str = ""
+    endpoints: list[dict] = field(default_factory=list)
+    schemas: list[dict] = field(default_factory=list)
 
 
 @dataclass

{webbee-0.3.52 → webbee-0.3.53}/src/webbee/intel/query.py

@@ -139,6 +139,53 @@ def search_code(svc, q, k: int = 20, kind=None, path_glob=None) -> dict:
     return _env(hits, has_more=len(fused) > k)
 
 
+def contract_slice(svc, query: str = "", kind: str = "all", k: int = 50) -> dict:
+    """Return bounded, source-proven endpoint/schema evidence for this repo.
+
+    Matching is deterministic substring search over route/method/handler/schema
+    names and fields. Semantic expansion belongs to `search_code`; this tool is
+    the exact contract projection used to ground cross-repo reasoning.
+    """
+    if svc is None or svc.index is None:
+        return {"ok": False, "content": "index not ready"}
+    q = (query or "").strip().lower()
+    wanted = (kind or "all").strip().lower()
+    limit = max(1, min(int(k or 50), 200))
+    items = []
+    for fi in svc.index.files.values():
+        if wanted in {"all", "endpoint"}:
+            for endpoint in fi.endpoints:
+                hay = " ".join((str(endpoint.get("method", "")),
+                                  str(endpoint.get("route", "")),
+                                  str(endpoint.get("handler", "")),
+                                  " ".join(endpoint.get("schema_refs") or ())))
+                if q and q not in hay.lower():
+                    continue
+                route = str(endpoint.get("route") or "")
+                method = str(endpoint.get("method") or "")
+                items.append({
+                    "id": f"{svc.repo_key}:endpoint:{method}:{route}:{endpoint.get('path')}:{endpoint.get('line')}",
+                    "title": f"{method} {route}", "kind": "endpoint",
+                    "repo_key": svc.repo_key, **endpoint,
+                })
+        if wanted in {"all", "schema"}:
+            for schema in fi.schemas:
+                hay = " ".join((str(schema.get("name", "")),
+                                  str(schema.get("schema_kind", "")),
+                                  str(schema.get("storage_name", "")),
+                                  " ".join(schema.get("fields") or ())))
+                if q and q not in hay.lower():
+                    continue
+                name = str(schema.get("name") or "")
+                items.append({
+                    "id": f"{svc.repo_key}:schema:{name}:{schema.get('path')}:{schema.get('line')}",
+                    "title": name, "kind": "schema",
+                    "repo_key": svc.repo_key, **schema,
+                })
+    items.sort(key=lambda x: (x["kind"], x["title"], x.get("path", ""), x.get("line", 0)))
+    return _env(items[:limit], has_more=len(items) > limit)
+
+
 def impact_of_change(svc, symbols) -> dict:
     if svc is None or svc.graph is None:
         return {"ok": False, "content": "index not ready"}

{webbee-0.3.52 → webbee-0.3.53}/src/webbee/intel/service.py

@@ -4,6 +4,7 @@ cold path (cache-load or full index) off the event loop, `.apply_changes()`
 does the cheap incremental re-parse the watcher drives, and `.repo_profile()`
 is the bounded summary handed to the brain."""
 from __future__ import annotations
+import hashlib
 import os
 import subprocess
 import threading
@@ -14,6 +15,7 @@ from webbee.intel.graph import CodeGraph
 from webbee.intel.models import ProjectIndex
 
 _MAX_PROFILE_SAMPLE = 20
+_MAX_CONTRACT_EVIDENCE = 5000
 _TEST_HINT_FILES = ("pytest.ini", "tox.ini", "package.json", "Makefile", "pyproject.toml")
 
 
@@ -242,13 +244,21 @@ class IntelService:
             files = list((self.index or ProjectIndex()).files.values())
             embedded_chunks = len(self.vectors.ids()) if self.vectors else 0
             vectors_ready = self.vectors_ready
+            content_hashes = sorted(fi.content_hash for fi in files if fi.content_hash)
+            endpoints = [e for fi in files for e in fi.endpoints]
+            schemas = [x for fi in files for x in fi.schemas]
         langs = Counter(fi.lang for fi in files if fi.lang != "other")
         kinds = Counter(s.kind for fi in files for s in fi.symbols)
         top = [f"{s.name} ({s.kind}) @ {s.path}:{s.start_line}"
                for fi in files[:200] for s in fi.symbols][:_MAX_PROFILE_SAMPLE]
         hints = [h for h in _TEST_HINT_FILES if os.path.exists(os.path.join(self.root, h))]
+        digest = hashlib.sha256("".join(content_hashes).encode()).hexdigest()
+        endpoints.sort(key=lambda x: (x.get("route", ""), x.get("method", ""), x.get("path", "")))
+        schemas.sort(key=lambda x: (x.get("name", ""), x.get("path", "")))
         return {
             "repo_key": self.repo_key,
+            "evidence_version": 1,
+            "content_digest": digest,
             "file_count": len(files),
             "languages": dict(langs),
             "symbol_kinds": dict(kinds),
@@ -258,4 +268,14 @@ class IntelService:
             "git_ref": self.git_ref[:12],
             "embedded_chunks": embedded_chunks,
             "vectors_ready": vectors_ready,
+            "endpoint_count": len(endpoints),
+            "schema_count": len(schemas),
+            # Account transport stays bounded; totals + cap metadata make a
+            # large repo's projection explicitly partial rather than quietly
+            # masquerading as the complete contract graph.
+            "contract_evidence_limit": _MAX_CONTRACT_EVIDENCE,
+            "contract_evidence_complete": (len(endpoints) <= _MAX_CONTRACT_EVIDENCE
+                                           and len(schemas) <= _MAX_CONTRACT_EVIDENCE),
+            "endpoints": endpoints[:_MAX_CONTRACT_EVIDENCE],
+            "schemas": schemas[:_MAX_CONTRACT_EVIDENCE],
         }

{webbee-0.3.52 → webbee-0.3.53}/src/webbee/intel/store.py

@@ -23,7 +23,7 @@ from webbee.intel.models import ProjectIndex, FileIndex, Symbol
 # schema 2 holds the OLD id format, so both the index and the vector cache
 # must be treated as a clean miss and rebuilt -- bumping the version is what
 # makes load()/load_vectors() do exactly that.
-SCHEMA_VERSION = 3
+SCHEMA_VERSION = 4
 
 
 def _path(cache_dir: str, repo_key: str) -> str:
@@ -35,6 +35,7 @@ def save(cache_dir: str, repo_key: str, index: ProjectIndex) -> None:
     os.makedirs(os.path.dirname(p), exist_ok=True)
     data = {"schema_version": SCHEMA_VERSION, "git_ref": index.git_ref,
             "files": {k: {"path": v.path, "lang": v.lang, "imports": v.imports, "refs": v.refs,
+                          "content_hash": v.content_hash, "endpoints": v.endpoints, "schemas": v.schemas,
                           "symbols": [asdict(s) for s in v.symbols]} for k, v in index.files.items()}}
     tmp = p + ".tmp"
     with open(tmp, "w", encoding="utf-8") as f:
@@ -59,7 +60,8 @@ def load(cache_dir: str, repo_key: str, git_ref: str) -> ProjectIndex | None:
         for k, v in (data.get("files") or {}).items():
             idx.files[k] = FileIndex(path=v["path"], lang=v["lang"],
                                      imports=v.get("imports", []), refs=v.get("refs", []),
-                                     symbols=[Symbol(**s) for s in v.get("symbols", [])])
+                                     content_hash=v.get("content_hash", ""), endpoints=v.get("endpoints", []),
+                                     schemas=v.get("schemas", []), symbols=[Symbol(**s) for s in v.get("symbols", [])])
         return idx
     except Exception:
         return None  # miss, corrupt, or wrong-shape -- never raise into the caller

webbee-0.3.53/src/webbee/native_files.py

@@ -0,0 +1,105 @@
+# Copyright (c) 2026 Imperal, Inc.
+# Licensed under the AGPL-3.0 License.
+"""Native document & image reading for the terminal agent.
+
+WHY THIS EXISTS
+---------------
+``read_file`` opened every path as UTF-8 text. A .docx, .pdf, .xlsx or a
+screenshot therefore did not read "badly" -- it raised UnicodeDecodeError and
+the agent got nothing at all, on every OS. Meanwhile the platform ALREADY
+owns a document service (the doc-extractor engine behind the *system* File
+Reader extension) that handles pdf/docx/xlsx/odt/pptx and images (OCR +
+vision). The gap was purely the missing bridge from a LOCAL path to that
+service.
+
+DESIGN (three rules that shape everything here)
+1. NO parsers on the client, none in the kernel. The engine is the single
+   source of truth for extraction -- reached ONLY through the system
+   file-reader extension, never by talking to the engine directly.
+2. Bytes must never travel through the model's context. The file is uploaded
+   out-of-band (the same door the panel dropzone, Telegram and clipboard
+   paste already use); the brain receives a compact TEXT window plus a
+   ``file_id`` handle it can page through -- so a 300-page PDF costs a
+   window, not a fortune in tokens.
+3. Honest degradation. No cloud reach, no plan, engine down, still indexing:
+   each returns a plain readable sentence saying what happened and what the
+   agent can do next -- never a stack trace, never a silent empty read.
+"""
+from __future__ import annotations
+
+import os
+
+# Formats the engine extracts. Value = mime type sent at upload; the ENGINE
+# re-sniffs content itself, so a wrong-but-plausible mime never decides the
+# outcome -- this is a hint, not a contract.
+DOCUMENT_MIMES: dict[str, str] = {
+    ".pdf": "application/pdf",
+    ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    ".doc": "application/msword",
+    ".odt": "application/vnd.oasis.opendocument.text",
+    ".rtf": "application/rtf",
+    ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    ".xls": "application/vnd.ms-excel",
+    ".ods": "application/vnd.oasis.opendocument.spreadsheet",
+    ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    ".ppt": "application/vnd.ms-powerpoint",
+    ".odp": "application/vnd.oasis.opendocument.presentation",
+    ".epub": "application/epub+zip",
+}
+
+IMAGE_MIMES: dict[str, str] = {
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".gif": "image/gif",
+    ".webp": "image/webp",
+    ".bmp": "image/bmp",
+    ".tif": "image/tiff",
+    ".tiff": "image/tiff",
+    ".heic": "image/heic",
+}
+
+# Product policy of the file-reader dropzone (providers/lifecycle.py).
+MAX_UPLOAD_BYTES = 100 * 1024 * 1024
+
+# .svg is genuine text (XML) -- readable as source. Keeping it OUT of the
+# native path means `read_file` on an icon still shows editable markup.
+_TEXT_FIRST = {".svg"}
+
+
+def classify(path: str) -> tuple[str, str]:
+    """(kind, mime) for a path. kind: 'document' | 'image' | 'text'.
+
+    Extension-based and deliberately conservative: anything unknown stays
+    'text' so the normal UTF-8 read (and its byte-exact edit contract) is
+    never disturbed. A binary that slips through is caught at read time by
+    the UnicodeDecodeError fallback, which routes here anyway.
+    """
+    ext = os.path.splitext(path)[1].lower()
+    if ext in _TEXT_FIRST:
+        return "text", ""
+    if ext in DOCUMENT_MIMES:
+        return "document", DOCUMENT_MIMES[ext]
+    if ext in IMAGE_MIMES:
+        return "image", IMAGE_MIMES[ext]
+    return "text", ""
+
+
+def guess_mime(path: str) -> str:
+    """Best-effort mime for a path the caller already knows is not text."""
+    ext = os.path.splitext(path)[1].lower()
+    return (DOCUMENT_MIMES.get(ext) or IMAGE_MIMES.get(ext)
+            or "application/octet-stream")
+
+
+def is_native(path: str) -> bool:
+    """True when the path is a document/image the engine should extract."""
+    return classify(path)[0] in ("document", "image")
+
+
+def human_size(n: float) -> str:
+    for unit in ("B", "KB", "MB", "GB"):
+        if n < 1024 or unit == "GB":
+            return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
+        n /= 1024.0
+    return f"{n:.1f} GB"