hexcli 2.8.0__py3-none-any.whl

hexcli/network.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""hexcli.network — Online detection and URL fetching for the fetch_url tool."""
+from __future__ import annotations
+
+import html.parser
+import ipaddress
+import re
+import socket
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+# ---------------------------------------------------------------------------
+# Online probe — cached 30 s
+# ---------------------------------------------------------------------------
+
+_online_result: bool | None = None
+_online_ts: float = 0.0
+_ONLINE_TTL = 30.0
+_PROBE_HOST = "8.8.8.8"
+_PROBE_PORT = 53
+_PROBE_TIMEOUT = 0.2  # 200 ms
+
+
+def is_online() -> bool:
+    """Return True if the internet appears reachable. Result is cached for 30 s."""
+    global _online_result, _online_ts
+    now = time.monotonic()
+    if _online_result is not None and now - _online_ts < _ONLINE_TTL:
+        return _online_result
+    try:
+        sock = socket.create_connection((_PROBE_HOST, _PROBE_PORT), timeout=_PROBE_TIMEOUT)
+        sock.close()
+        result = True
+    except OSError:
+        result = False
+    _online_result = result
+    _online_ts = now
+    return result
+
+
+# ---------------------------------------------------------------------------
+# URL security check
+# ---------------------------------------------------------------------------
+
+def _is_blocked_url(url: str) -> str | None:
+    """Return an error string if the URL should be blocked, None if safe to fetch."""
+    try:
+        parsed = urllib.parse.urlsplit(url)
+    except Exception:
+        return "fetch_url: invalid URL."
+
+    scheme = (parsed.scheme or "").lower()
+    if scheme not in ("http", "https"):
+        return f"fetch_url: only http/https URLs are allowed (got scheme {scheme!r})."
+
+    host = (parsed.hostname or "").lower()
+    if not host:
+        return "fetch_url: URL has no host."
+    if host == "localhost":
+        return "fetch_url: private/local addresses are blocked."
+
+    try:
+        addr = ipaddress.ip_address(host)
+        if addr.is_private or addr.is_loopback or addr.is_link_local:
+            return f"fetch_url: private address {host!r} is blocked."
+    except ValueError:
+        pass  # hostname — let DNS resolve it; private hostnames can't be easily checked here
+
+    return None
+
+
+# ---------------------------------------------------------------------------
+# HTML stripping
+# ---------------------------------------------------------------------------
+
+class _StripHTML(html.parser.HTMLParser):
+    _SKIP = frozenset({"script", "style", "nav", "header", "footer", "aside"})
+    _BLOCK = frozenset({"p", "br", "div", "li", "h1", "h2", "h3", "h4", "h5", "h6", "tr", "td", "section", "article"})
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self._parts: list[str] = []
+        self._skip_depth = 0
+
+    def handle_starttag(self, tag: str, attrs: object) -> None:
+        if tag in self._SKIP:
+            self._skip_depth += 1
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in self._SKIP:
+            self._skip_depth = max(0, self._skip_depth - 1)
+        elif tag in self._BLOCK and not self._skip_depth:
+            self._parts.append("\n")
+
+    def handle_data(self, data: str) -> None:
+        if not self._skip_depth:
+            self._parts.append(data)
+
+    def get_text(self) -> str:
+        text = "".join(self._parts)
+        text = re.sub(r"\n{3,}", "\n\n", text)
+        text = re.sub(r"[ \t]+", " ", text)
+        return text.strip()
+
+
+# ---------------------------------------------------------------------------
+# fetch_url
+# ---------------------------------------------------------------------------
+
+def fetch_url(url: str, max_chars: int = 2500) -> str:
+    """Fetch a URL and return plain text. Blocks private IPs and non-HTTP/HTTPS schemes."""
+    block = _is_blocked_url(url)
+    if block:
+        return block
+
+    if not is_online():
+        return "fetch_url: no network connection available."
+
+    try:
+        req = urllib.request.Request(
+            url,
+            headers={"User-Agent": "hexcli/1.7 (local agent; +https://github.com/NathanL15/Hex-CLI)"},
+        )
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            content_type = resp.headers.get("Content-Type", "")
+            raw = resp.read(max_chars * 8)
+    except urllib.error.HTTPError as exc:
+        return f"fetch_url: HTTP {exc.code} {exc.reason} — {url}"
+    except urllib.error.URLError as exc:
+        return f"fetch_url: could not reach {url!r}: {exc.reason}"
+    except Exception as exc:
+        return f"fetch_url: error — {exc}"
+
+    try:
+        text_raw = raw.decode("utf-8", errors="replace")
+    except Exception:
+        text_raw = raw.decode("latin-1", errors="replace")
+
+    if "html" in content_type.lower() or text_raw.strip().lower().startswith("<!"):
+        parser = _StripHTML()
+        try:
+            parser.feed(text_raw)
+            text = parser.get_text()
+        except Exception:
+            text = text_raw
+    else:
+        text = text_raw
+
+    if len(text) > max_chars:
+        text = text[:max_chars] + f"\n...[truncated to {max_chars} chars]"
+
+    return text or f"(empty response from {url})"

hexcli/parsing.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+"""hexcli.parsing — the wire protocol's text side, lifted out of agent.py.
+
+Everything here is pure text/JSON interpretation with no agent state: trim
+helpers, the query normalizers, <think>-stripping, and the JSON-action parser
+that turns a model response into a dispatchable action. TOOL_NAMES lives here
+because it is the action vocabulary this parser interprets; agent.py re-binds
+it (and everything else) by name, so `sa.parse_agent_action` and friends keep
+working for every existing caller and eval.
+
+Split stage 1 (docs/V2X_ROADMAP.md, "The Split"). Function bodies are moved
+verbatim — behavior changes do not belong in split commits.
+"""
+from __future__ import annotations
+
+import json
+import re
+import shutil
+from typing import Any
+
+# ruff is an optional hard dependency: if present, lint_code is registered as a
+# live tool and injected into the system prompt. If absent, the tool simply does
+# not appear — no fallback needed since verify_syntax covers the critical path.
+_RUFF: str | None = shutil.which("ruff")
+
+TOOL_NAMES = frozenset({
+    "run_command", "read_file", "edit_file", "write_file",
+    "append_file", "list_directory", "search_files", "find_files",
+    "verify_syntax", "search_memory", "run_code",
+    "fetch_url", "batch", "delegate",
+    *(["lint_code"] if _RUFF else []),
+})
+
+
+# ---------------------------------------------------------------------------
+# Text utilities
+# ---------------------------------------------------------------------------
+
+def trim_text(text: str, limit: int) -> str:
+    """Head-only truncation. Use trim_tool_output for command/tool results."""
+    if len(text) <= limit:
+        return text
+    return text[:limit] + f"\n...[truncated to {limit} chars]"
+
+
+def trim_tool_output(text: str, limit: int) -> str:
+    """Head+tail truncation for tool results.
+
+    Command output carries its verdict at the END — exit summaries, stack
+    traces, assertion messages. v1.7's head-only trim hid exactly the part
+    the model needed to recover from a failure, so keep both ends.
+    """
+    if len(text) <= limit:
+        return text
+    head = int(limit * 0.6)
+    tail = max(0, limit - head)
+    omitted = len(text) - head - tail
+    return f"{text[:head]}\n...[{omitted} chars omitted]...\n{text[-tail:]}" if tail else trim_text(text, limit)
+
+
+def normalize_text(value: str) -> str:
+    return re.sub(r"\s+", " ", value.strip().lower())
+
+
+def is_help_request(query: str) -> bool:
+    normalized = re.sub(r"[?!.,]+", "", normalize_text(query))
+    return normalized in {"help", "/help", "what can you do", "what is this", "how do i use this"}
+
+
+def is_small_talk(query: str) -> bool:
+    normalized = re.sub(r"[?!.,]+", "", normalize_text(query))
+    return normalized in {"hi", "hello", "hey", "whats up", "what is up", "yo"}
+
+
+def local_meta_response(query: str, config: dict[str, Any]) -> str | None:
+    normalized = re.sub(r"[?!.,]+", "", normalize_text(query))
+    model = str(config.get("model", "unknown")).strip() or "unknown"
+    backend = str(config.get("backend", "ollama")).strip()
+    label = f"{model} via {'Ollama' if backend == 'ollama' else 'local OpenAI-compatible endpoint'}"
+    if any(p in normalized for p in ("what model are you", "which model are you", "what llm")):
+        return f"Using {label}."
+    if normalized in {"who are you", "what are you"}:
+        return f"Local coding and system agent powered by {label}."
+    return None
+
+
+def strip_thinking(text: str) -> str:
+    """Remove <think>...</think> blocks emitted by reasoning models like deepseek-r1."""
+    return re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL).strip()
+
+
+# ---------------------------------------------------------------------------
+# Parsing
+# ---------------------------------------------------------------------------
+
+def parse_json_object(raw_text: str) -> dict[str, Any] | None:
+    text = strip_thinking(raw_text).strip()
+    if not text:
+        return None
+    # Direct parse
+    try:
+        parsed = json.loads(text)
+        if isinstance(parsed, dict):
+            return parsed
+    except json.JSONDecodeError:
+        pass
+    # Strip markdown fences
+    stripped = re.sub(r"^```[a-zA-Z]*\s*|```\s*$", "", text, flags=re.MULTILINE).strip()
+    try:
+        parsed = json.loads(stripped)
+        if isinstance(parsed, dict):
+            return parsed
+    except json.JSONDecodeError:
+        pass
+    # Extract the FIRST complete {...} object by brace balancing.
+    #
+    # This used to be a greedy re.search(r"\{.*\}"), which spans from the first
+    # brace to the LAST one. When the model emits several actions in a row —
+    # {"action":"edit_file",…},{"action":"verify_syntax",…},{"action":"run_code",…}
+    # — that match is not valid JSON, so the whole response was discarded, the
+    # identical retry was issued up to 3×, and the turn ended with no tool call
+    # at all. Measured 2026-07-30: this is what actually killed uc1-t4/t5/t6
+    # (0/3 each), NOT a context-length cliff. Batching is a natural response to
+    # rules that prescribe an edit→verify→run sequence, so take the first
+    # action and let the loop drive the rest.
+    for candidate in _iter_json_objects(text):
+        try:
+            parsed = json.loads(candidate)
+            if isinstance(parsed, dict):
+                return parsed
+        except json.JSONDecodeError:
+            continue
+    return None
+
+
+def _iter_json_objects(text: str):
+    """Yield complete brace-balanced {...} substrings, in order.
+
+    String-literal aware, so braces inside JSON strings never affect depth.
+    """
+    depth = 0
+    start = -1
+    in_string = False
+    escaped = False
+    for i, ch in enumerate(text):
+        if in_string:
+            if escaped:
+                escaped = False
+            elif ch == "\\":
+                escaped = True
+            elif ch == '"':
+                in_string = False
+            continue
+        if ch == '"':
+            in_string = True
+        elif ch == "{":
+            if depth == 0:
+                start = i
+            depth += 1
+        elif ch == "}":
+            if depth > 0:
+                depth -= 1
+                if depth == 0 and start >= 0:
+                    yield text[start:i + 1]
+                    start = -1
+
+
+def parse_agent_action(raw_text: str) -> dict[str, Any]:
+    parsed = parse_json_object(raw_text)
+    if isinstance(parsed, dict):
+        action = str(parsed.get("action", "")).strip().lower()
+        args = parsed.get("args")
+        args = args if isinstance(args, dict) else {}
+
+        if action in TOOL_NAMES:
+            return {"action": "tool", "tool": action, "args": args}
+        if action == "tool":
+            tool = str(parsed.get("tool", "")).strip()
+            return {"action": "tool", "tool": tool, "args": args}
+        if action == "finish":
+            return {"action": "finish", "message": str(parsed.get("message") or "").strip()}
+
+        tool = str(parsed.get("tool", "")).strip()
+        if tool in TOOL_NAMES:
+            return {"action": "tool", "tool": tool, "args": args}
+
+        message = str(parsed.get("message") or "").strip()
+        if message:
+            # Valid JSON, but the action name (if any) matched nothing. Tag it
+            # so the loop can distinguish "model typo'd a tool name" (worth a
+            # retry naming the bad action) from an intended finish.
+            return {"action": "finish", "message": message,
+                    "fallback": "unknown-action", "bad_action": action}
+
+    # No parseable JSON anywhere: the message is just the raw prose. This is
+    # the deliberate direct-answer path for knowledge questions — but the loop
+    # retries it when the text shows signs of an ATTEMPTED action (see
+    # _looks_like_botched_action), because "prose instead of action" was the
+    # uc1-t5/t6 failure mode.
+    return {"action": "finish", "message": strip_thinking(raw_text).strip(),
+            "fallback": "prose"}
+
+
+def _looks_like_botched_action(raw_text: str) -> bool:
+    """Does an unparseable response look like it TRIED to be an action?
+
+    Braces or code fences mean attempted JSON; a tool name means attempted
+    tool use. Pure prose with none of those is accepted as an implicit finish
+    — that path is load-bearing for direct answers, so this must stay
+    conservative about flagging it.
+    """
+    text = strip_thinking(raw_text)
+    if "{" in text or "```" in text:
+        return True
+    return any(name in text for name in TOOL_NAMES)

hexcli/paths.py

@@ -0,0 +1,127 @@
+"""hexcli.paths — where the app keeps its things.
+
+Two homes:
+
+* the package (``hexcli/``), read-only once installed: code and the icon;
+* the data directory, ``~/.shellai``: the user config, the runtime config
+  the launcher writes, the embedding model for memory, the server log,
+  the session history, chat logs, memory stores, input history.
+
+A git checkout is also recognised (``pyproject.toml`` next to the
+package). It never wins over the data directory, but files that older
+versions kept in the checkout — ``shellai.json``, ``history.json``,
+``onnx/``, a downloaded ``npurun-arm64.exe`` — are still found there, so
+a developer's checkout keeps working and a first run migrates the
+history file rather than losing it.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+
+PACKAGE_DIR = Path(__file__).resolve().parent
+CHECKOUT_DIR: Path | None = (
+    PACKAGE_DIR.parent if (PACKAGE_DIR.parent / "pyproject.toml").exists() else None
+)
+DATA_DIR_NAME = ".shellai"
+
+EMBEDDING_MODEL_FILE = "model_qint8_arm64.onnx"
+EMBEDDING_TOKENIZER_FILE = "tokenizer.json"
+EMBEDDING_BASE_URL = "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main"
+NPURUN_ASSET = "npurun-arm64.exe"
+
+
+def data_dir(create: bool = True) -> Path:
+    """``~/.shellai`` (``HEXCLI_HOME`` overrides it, for tests and sandboxes)."""
+    override = os.environ.get("HEXCLI_HOME")
+    d = Path(override).expanduser() if override else Path.home() / DATA_DIR_NAME
+    if create:
+        try:
+            d.mkdir(parents=True, exist_ok=True)
+        except OSError:
+            pass
+    return d
+
+
+def _first_existing(*candidates: Path | None) -> Path | None:
+    for c in candidates:
+        if c is not None and c.exists():
+            return c
+    return None
+
+
+def _migrated(name: str) -> Path:
+    """``~/.shellai/<name>``, copied once from a checkout's ``<name>`` when
+    only the checkout has it (older versions kept these files there)."""
+    home = data_dir() / name
+    if not home.exists() and CHECKOUT_DIR is not None:
+        old = CHECKOUT_DIR / name
+        if old.exists():
+            try:
+                shutil.copy2(old, home)
+            except OSError:
+                return old
+    return home
+
+
+def user_config_path() -> Path:
+    """``~/.shellai/shellai.json`` (migrated once from a checkout's copy)."""
+    return _migrated("shellai.json")
+
+
+def runtime_config_path() -> Path:
+    """The config the launcher writes for the npurun server (backend wiring)."""
+    return _migrated("shellai_npurun.json")
+
+
+def history_path() -> Path:
+    """Session history. Migrated once from a checkout's ``history.json``."""
+    return _migrated("history.json")
+
+
+def npurun_log_path() -> Path:
+    return data_dir() / "npurun_server.log"
+
+
+def embedding_dir(for_download: bool = False) -> Path:
+    """Where the MiniLM files live: ``~/.shellai/onnx``, or a checkout's
+    ``onnx/`` when only that has them. ``for_download`` asks for the place
+    to put them."""
+    home = data_dir() / "onnx"
+    if for_download:
+        return home
+    checkout = CHECKOUT_DIR / "onnx" if CHECKOUT_DIR else None
+    for d in (home, checkout):
+        if d is not None and (d / EMBEDDING_MODEL_FILE).exists():
+            return d
+    return home
+
+
+def embedding_model_path() -> Path:
+    return embedding_dir() / EMBEDDING_MODEL_FILE
+
+
+def embedding_tokenizer_path() -> Path:
+    return embedding_dir() / EMBEDDING_TOKENIZER_FILE
+
+
+def npurun_bin_dir() -> Path:
+    """Where a downloaded npurun binary goes (``~/.shellai/bin``)."""
+    return data_dir() / "bin"
+
+
+def npurun_download_candidates() -> list[Path]:
+    """Downloaded binaries, newest home first, then a checkout's copy."""
+    out = [npurun_bin_dir() / NPURUN_ASSET]
+    if CHECKOUT_DIR is not None:
+        out.append(CHECKOUT_DIR / NPURUN_ASSET)
+    return out
+
+
+def icon_path() -> Path:
+    return PACKAGE_DIR / "assets" / "hexcli.ico"
+
+
+def icon_png_path() -> Path:
+    return PACKAGE_DIR / "assets" / "hexcli.png"