code-jig 0.12.0__py3-none-any.whl

codejig/sources.py

@@ -0,0 +1,248 @@
+import contextlib
+import errno
+import glob
+import http.client
+import ipaddress
+import json
+import os
+import re
+import socket
+import sys
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from html.parser import HTMLParser
+
+MAX_BYTES = 3_000_000
+TIMEOUT = 20
+USER_AGENT = "Mozilla/5.0 (compatible; code-jig check-sources)"
+BLOCKED = {401, 403, 429, 451}
+SKIP_TAGS = {"script", "style", "noscript", "template"}
+TEXT_TYPES = ("text/", "application/xhtml", "application/xml", "application/json")
+FAILING = {"dead", "quote-missing", "bad-url", "no-quote", "malformed"}
+
+
+@dataclass
+class Claim:
+    where: str
+    claim: str = ""
+    url: str = ""
+    quote: str = ""
+    verdict: str = ""
+    detail: str = ""
+
+
+class _Text(HTMLParser):
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.parts, self.skip = [], 0
+
+    def handle_starttag(self, tag, _attrs):
+        self.skip += tag in SKIP_TAGS
+        self.parts.append(" ")
+
+    def handle_endtag(self, tag):
+        self.skip -= tag in SKIP_TAGS and self.skip > 0
+        self.parts.append(" ")
+
+    def handle_data(self, data):
+        if not self.skip:
+            self.parts.append(data)
+
+
+def normalize(text: str) -> str:
+    return " ".join(re.sub(r"[\W_]+", " ", text.casefold()).split())
+
+
+def page_text(body: bytes, ctype: str) -> str:
+    charset = re.search(r"charset=([\w-]+)", ctype)
+    text = body.decode(charset.group(1) if charset else "utf-8", errors="replace")
+    if "html" in ctype or "xml" in ctype:
+        parser = _Text()
+        try:
+            parser.feed(text)
+        except AssertionError:
+            pass
+        text = "".join(parser.parts)
+    return normalize(text)
+
+
+def is_public_ip(ip: str) -> bool:
+    return ipaddress.ip_address(ip.split("%")[0]).is_global
+
+
+def is_public(host: str) -> bool | None:
+    """True when every address the host resolves to is public, False when any is not,
+    None when it does not resolve here."""
+    try:
+        infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
+    except (OSError, UnicodeError):
+        return None
+    return all(is_public_ip(info[4][0]) for info in infos)
+
+
+def _target(url: str) -> tuple[str, str] | None:
+    """(scheme, host) exactly as urllib and http.client will parse them for the connection."""
+    try:
+        req = urllib.request.Request(url)
+        return req.type.lower(), http.client.HTTPConnection(req.host).host
+    except (ValueError, http.client.InvalidURL):
+        return None
+
+
+def refused(url: str) -> str | None:
+    """Why `url` must not be requested, or None. Through a proxy, a host that does not resolve here
+    is refused: the proxy may resolve internal names this machine cannot."""
+    target = _target(url)
+    if target is None or target[0] not in ("http", "https") or not target[1]:
+        return "only http and https are checked"
+    public = is_public(target[1])
+    if public is False:
+        return f"{target[1]} is a private or local address"
+    if public is None and proxied(url):
+        return f"cannot confirm {target[1]} is public: it does not resolve here and a proxy would fetch it"
+    return None
+
+
+def proxied(url: str) -> bool:
+    target = _target(url)
+    proxies = urllib.request.getproxies()
+    return bool(target) and target[0] in proxies and not urllib.request.proxy_bypass(target[1])
+
+
+def _check_peer(conn: http.client.HTTPConnection) -> None:
+    if conn._tunnel_host:
+        return
+    ip = conn.sock.getpeername()[0]
+    if not is_public_ip(ip):
+        conn.close()
+        raise OSError(f"refused: {ip} is a private or local address")
+
+
+class _HTTPConnection(http.client.HTTPConnection):
+    def connect(self):
+        super().connect()
+        _check_peer(self)
+
+
+class _HTTPSConnection(http.client.HTTPSConnection):
+    def connect(self):
+        super().connect()
+        _check_peer(self)
+
+
+class _HTTPHandler(urllib.request.HTTPHandler):
+    def http_open(self, req):
+        return self.do_open(_HTTPConnection, req)
+
+
+class _HTTPSHandler(urllib.request.HTTPSHandler):
+    def https_open(self, req):
+        return self.do_open(_HTTPSConnection, req, context=self._context)
+
+
+class _Redirects(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        reason = refused(newurl)
+        if reason:
+            raise urllib.error.URLError(f"redirect refused: {reason}")
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+DIRECT = urllib.request.build_opener(urllib.request.ProxyHandler({}), _HTTPHandler, _HTTPSHandler, _Redirects)
+VIA_PROXY = urllib.request.build_opener(_Redirects)
+
+
+def fetch(url: str) -> tuple[str, str]:
+    """(verdict, page text or detail) for one URL. Only public http and https addresses are requested,
+    redirects included; without a proxy, the address the socket actually connected to is checked too."""
+    reason = refused(url)
+    if reason:
+        return "bad-url", reason
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "text/html,text/plain;q=0.9,*/*;q=0.5"})
+    try:
+        with (VIA_PROXY if proxied(url) else DIRECT).open(req, timeout=TIMEOUT) as r:
+            ctype = r.headers.get("Content-Type", "")
+            if not ctype.startswith(TEXT_TYPES):
+                return "unreadable", ctype or "unknown type"
+            return "ok", page_text(r.read(MAX_BYTES), ctype)
+    except urllib.error.HTTPError as e:
+        e.close()
+        return ("blocked" if e.code in BLOCKED else "dead"), f"HTTP {e.code}"
+    except (urllib.error.URLError, OSError, ValueError) as e:
+        return "dead", str(getattr(e, "reason", e))
+
+
+def expand(patterns: list[str]) -> list[str]:
+    """File names with shell patterns expanded here, so PowerShell and cmd behave like a POSIX shell."""
+    out = []
+    for p in patterns:
+        hits = (sorted(glob.glob(p)) or ([p] if os.path.exists(p) else [])) if glob.has_magic(p) else [p]
+        if not hits:
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), p)
+        out += hits
+    return out
+
+
+def _pre_verdict(c: Claim) -> None:
+    if not c.url:
+        c.verdict, c.detail = "malformed", "no url"
+    elif not c.url.lower().startswith(("http://", "https://")):
+        c.verdict, c.detail = "bad-url", "only http and https are checked"
+    elif not normalize(c.quote):
+        c.verdict, c.detail = "no-quote", "a claim needs an exact quote with words in it"
+
+
+def _parse_line(where: str, line: str) -> Claim:
+    c = Claim(where)
+    try:
+        row = json.loads(line)
+        c.claim, c.url, c.quote = (str(row.get(k) or "").strip() for k in ("claim", "url", "quote"))
+    except (ValueError, AttributeError):
+        c.verdict, c.detail = "malformed", "not a JSON object"
+        return c
+    _pre_verdict(c)
+    return c
+
+
+def load(files: list[str]) -> list[Claim]:
+    claims = []
+    for name in expand(files):
+        with contextlib.nullcontext(sys.stdin) if name == "-" else open(name, encoding="utf-8-sig") as f:
+            claims += [_parse_line(f"{name}:{n}", line) for n, line in enumerate(f, 1) if line.strip()]
+    return claims
+
+
+def check(claims: list[Claim]) -> list[Claim]:
+    todo = [c for c in claims if not c.verdict]
+    urls = list(dict.fromkeys(c.url for c in todo))
+    with ThreadPoolExecutor(max_workers=8) as pool:
+        pages = dict(zip(urls, pool.map(fetch, urls), strict=True))
+    for c in todo:
+        verdict, text = pages[c.url]
+        if verdict != "ok":
+            c.verdict, c.detail = verdict, text
+        elif normalize(c.quote) in text:
+            c.verdict = "ok"
+        else:
+            c.verdict, c.detail = "quote-missing", "the page does not contain the quote"
+    return claims
+
+
+def cli(args) -> int:
+    if os.environ.get("CODEX_SANDBOX_NETWORK_DISABLED") == "1":
+        print("code-jig: network is disabled in this sandbox; run check-sources with escalation", file=sys.stderr)
+        return 2
+    try:
+        claims = check(load(args.files))
+    except OSError as e:
+        print(f"code-jig: {e}", file=sys.stderr)
+        return 2
+    counts: dict[str, int] = {}
+    for c in claims:
+        counts[c.verdict] = counts.get(c.verdict, 0) + 1
+        note = f" ({c.detail})" if c.detail else ""
+        print(f"{c.verdict:13} {c.where}  {c.claim[:70]}  {c.url}{note}")
+    print(f"{len(claims)} claims: " + ", ".join(f"{n} {v}" for v, n in sorted(counts.items(), key=lambda kv: kv[0] != "ok")))
+    return 1 if FAILING & counts.keys() else 0

codejig/tiers.py

@@ -0,0 +1,63 @@
+import json
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Tier:
+    name: str
+    model: str
+    effort: str
+
+
+def read_toml(path: Path) -> dict:
+    try:
+        return tomllib.loads(path.read_text(encoding="utf-8-sig"))
+    except FileNotFoundError:
+        raise ValueError(f"{path}: not found") from None
+    except (tomllib.TOMLDecodeError, UnicodeDecodeError) as e:
+        raise ValueError(f"{path}: {e}") from e
+
+
+def _strings(path: Path, data: dict) -> dict:
+    bad = [k for k, v in data.items() if not isinstance(v, str) or not v.strip()]
+    if bad:
+        raise ValueError(f"{path}: {', '.join(bad)} must be a non-empty string")
+    return data
+
+
+def load(core: Path, overlay_root: Path) -> list[Tier]:
+    efforts_path = core / "core" / "tiers.toml"
+    models_path = overlay_root / "profiles.toml"
+    efforts = _strings(efforts_path, read_toml(efforts_path))
+    models = _strings(models_path, read_toml(models_path))
+    missing = [t for t in efforts if t not in models]
+    if missing:
+        raise ValueError(f"{models_path}: no model for tier {', '.join(missing)}")
+    unknown = [t for t in models if t not in efforts]
+    if unknown:
+        raise ValueError(f"{models_path}: unknown tier {', '.join(unknown)}; tiers are {', '.join(efforts)}")
+    return [Tier(name, models[name], effort) for name, effort in efforts.items()]
+
+
+def extra_policy(overlay_root: Path) -> str:
+    path = overlay_root / "guardian-policy.md"
+    return path.read_text(encoding="utf-8-sig").strip() if path.is_file() else ""
+
+
+def toml_str(s: str) -> str:
+    return json.dumps(s, ensure_ascii=False).replace("\x7f", "\\u007f")
+
+
+def profile_body(tier: Tier, policy: str, compact: int | None = None) -> str:
+    lines = [
+        f"model = {toml_str(tier.model)}",
+        f"model_reasoning_effort = {toml_str(tier.effort)}",
+        'approvals_reviewer = "auto_review"',
+    ]
+    if compact is not None:
+        lines.append(f"model_auto_compact_token_limit = {compact}")
+    if policy:
+        lines += ["", "[auto_review]", f"extra_policy = {toml_str(policy)}"]
+    return "\n".join(lines) + "\n"

codejig/tokens.py

@@ -0,0 +1,118 @@
+import sqlite3
+import sys
+from collections import Counter, defaultdict
+from datetime import UTC, datetime, timedelta
+
+from codejig import collect
+
+TWO_WORD = {"git", "npm", "pnpm", "yarn", "cargo", "go", "uv", "dotnet", "docker", "kubectl", "gh"}
+POLL = ("sleep", "wait")
+
+
+def prefix(cmd: str) -> str:
+    words = [w for w in cmd.split() if not ("=" in w and not w.startswith("-") and w.split("=", 1)[0].isidentifier())]
+    if not words:
+        return ""
+    first = words[0].replace("\\", "/").rsplit("/", 1)[-1]
+    if first in TWO_WORD and len(words) > 1 and not words[1].startswith("-"):
+        return f"{first} {words[1]}"
+    return first
+
+
+def _tally(rows: list, comps: Counter) -> tuple[dict, dict, dict, set]:
+    by_kind: dict[str, int] = defaultdict(int)
+    models: dict[str, list[int]] = defaultdict(lambda: [0, 0])
+    sessions: dict[str, dict] = {}
+    reviews = set()
+    for thread, kind, parent, turn, model, inp, cached, out in rows:
+        by_kind[kind] += inp
+        models[model or "(unknown)"][0] += inp
+        models[model or "(unknown)"][1] += out
+        owner = parent if kind != "main" and parent else thread
+        s = sessions.setdefault(owner, {"id": owner, "turns": 0, "input": 0, "cached": 0, "guardian_input": 0,
+                                        "subagent_input": 0, "compactions": comps.get(owner, 0), "polling": 0})
+        if kind == "guardian":
+            s["guardian_input"] += inp
+            reviews.add((thread, turn))
+        elif kind == "subagent":
+            s["subagent_input"] += inp
+        else:
+            s["input"] += inp
+            s["cached"] += cached
+            s["turns"] += 1
+    return by_kind, models, sessions, reviews
+
+
+def _count_polling(calls: list, sessions: dict) -> None:
+    for thread, cmd, _ in calls:
+        if thread in sessions and cmd.lstrip().split(" ", 1)[0] in POLL:
+            sessions[thread]["polling"] += 1
+
+
+def summary(con: sqlite3.Connection, since: str, top: int = 10, long_turns: int = 300) -> dict:
+    rows = con.execute("SELECT thread_id, kind, parent_id, turn_id, model, input, cached, output FROM turn_usage WHERE ts >= ?", (since,)).fetchall()
+    calls = con.execute("SELECT thread_id, cmd, escalated FROM calls WHERE ts >= ?", (since,)).fetchall()
+    comps = Counter(t for (t,) in con.execute("SELECT thread_id FROM compactions WHERE ts >= ?", (since,)))
+    by_kind, models, sessions, reviews = _tally(rows, comps)
+    _count_polling(calls, sessions)
+    total = sum(by_kind.values())
+    guardian = by_kind.get("guardian", 0)
+    ranked = sorted(sessions.values(), key=lambda s: s["input"] + s["guardian_input"] + s["subagent_input"], reverse=True)
+    return {
+        "input": total,
+        "cached": sum(r[6] for r in rows),
+        "output": sum(r[7] for r in rows),
+        "by_kind": dict(by_kind),
+        "guardian_share": guardian / total if total else 0.0,
+        "reviews": len(reviews),
+        "input_per_review": guardian / len(reviews) if reviews else 0,
+        "escalations": Counter(prefix(c) for _, c, e in calls if e).most_common(),
+        "sessions": ranked[:top],
+        "over_long": sum(1 for s in sessions.values() if s["turns"] > long_turns),
+        "models": sorted(((m, v[0], v[1]) for m, v in models.items()), key=lambda r: r[1], reverse=True),
+        "quarantined": con.execute("SELECT COUNT(*) FROM quarantine").fetchone()[0],
+    }
+
+
+def _safe(text: str) -> str:
+    return "".join(ch if ch.isprintable() else f"\\x{ord(ch):02x}" for ch in text)
+
+
+def _pct(a: int, b: int) -> str:
+    return f"{100 * a / b:.1f}%" if b else "-"
+
+
+def render(s: dict, floor: float) -> list[str]:
+    out = [
+        f"Input {s['input']:,}  cached {_pct(s['cached'], s['input'])}  output {s['output']:,}",
+        f"Guardian share of input: {100 * s['guardian_share']:.1f}%  reviews {s['reviews']}  input per review {s['input_per_review']:,.0f}",
+        f"Sessions over 300 model requests: {s['over_long']}",
+        "",
+        "Top sessions (own input, cache hit, model requests, compactions, polling calls, Guardian input, subagent input):",
+    ]
+    for x in s["sessions"]:
+        hit = 100 * x["cached"] / x["input"] if x["input"] else 100.0
+        flag = "  LOW" if x["input"] and hit < floor else ""
+        out.append(f"  {_safe(x['id'])}  {x['input']:,}  {hit:.1f}%{flag}  {x['turns']}  {x['compactions']}  {x['polling']}  {x['guardian_input']:,}  {x['subagent_input']:,}")
+    out += ["", "Escalations sent to Guardian, by command:"]
+    out += [f"  {_safe(p) or '(empty)'}  {n}" for p, n in s["escalations"]] or ["  none"]
+    out += ["", "By model (input, output):"]
+    out += [f"  {_safe(m)}  {i:,}  {o:,}" for m, i, o in s["models"]] or ["  none"]
+    if s["quarantined"]:
+        out += ["", f"warning: {s['quarantined']} rollout lines quarantined; see the quarantine table"]
+    return out
+
+
+def cli(args) -> int:
+    path = collect.db_path()
+    if not path.is_file():
+        print(f"code-jig: {path} not found; run code-jig collect", file=sys.stderr)
+        return 2
+    con = collect.connect(path)
+    try:
+        since = (datetime.now(UTC) - timedelta(days=args.days)).strftime("%Y-%m-%dT%H:%M:%SZ")
+        for line in render(summary(con, since, args.top), args.cache_floor):
+            print(line)
+    finally:
+        con.close()
+    return 0

codejig/trace.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import codecs
+import json
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+from codejig.analyze import printable, source_files, utf8_stdout
+
+TEST_PATH = re.compile(
+    r"(^|/)(tests?|__tests__|specs?)/"
+    r"|\.(test|spec)\.[cm]?[jt]sx?$"
+    r"|(^|/)test_[^/]*\.py$"
+    r"|_test\.py$"
+    r"|Tests?\.cs$"
+    r"|(^|/)[^/]*\.Tests?/"
+)
+ID_CHARS = r"[A-Za-z0-9_]+(?:[.-][A-Za-z0-9_]+)*"
+MAX_BYTES = 2_000_000
+LINE_ID = re.compile(rf"^\s*(?:[-*+]\s+|\d+[.)]\s+)?#?(?:REQ-)?({ID_CHARS})", re.IGNORECASE)
+
+
+@dataclass(frozen=True)
+class Result:
+    covered: dict[str, list[str]]
+    missing: list[str]
+    unknown: dict[str, list[str]]
+
+
+def _norm(value) -> str:
+    text = str(value).strip().lstrip("#")
+    return text[4:] if text.lower().startswith("req-") else text
+
+
+def _json_ids(data) -> list:
+    if isinstance(data, dict):
+        data = data.get("workItems") or data.get("value") or ([data] if "id" in data else [])
+    if isinstance(data, (int, str)) and not isinstance(data, bool):
+        return [data]
+    if not isinstance(data, list):
+        return []
+    return [item.get("id") if isinstance(item, dict) else item for item in data if not isinstance(item, (list, bool))]
+
+
+def read_ids(text: str) -> list[str]:
+    text = text.lstrip("\ufeff")
+    try:
+        raw = _json_ids(json.loads(text))
+    except ValueError:
+        if text.lstrip().startswith(("[", "{")):
+            raise ValueError("looks like JSON but does not parse") from None
+        raw = [m.group(1) for m in map(LINE_ID.match, text.splitlines()) if m]
+    ids = [_norm(v) for v in raw if v is not None and _norm(v)]
+    return list(dict.fromkeys(ids))
+
+
+def _regex(pattern: str, id_part: str) -> re.Pattern:
+    if "{id}" not in pattern:
+        raise ValueError("pattern must contain {id}")
+    head, _, tail = pattern.partition("{id}")
+    return re.compile(rf"(?<![\w-]){re.escape(head)}{id_part}{re.escape(tail)}(?!\w|[.-]\w)", re.IGNORECASE)
+
+
+def trace(root: Path, ids: list[str], pattern: str = "REQ-{id}") -> Result:
+    wanted = {i.lower(): i for i in ids}
+    finder = _regex(pattern, f"({ID_CHARS})")
+    covered: dict[str, list[str]] = {}
+    unknown: dict[str, list[str]] = {}
+    for name in source_files(root):
+        if not TEST_PATH.search(name):
+            continue
+        f = root / name
+        if f.stat().st_size > MAX_BYTES:
+            continue
+        text = f.read_text(encoding="utf-8", errors="replace")
+        for found in dict.fromkeys(m.group(1) for m in finder.finditer(text)):
+            target = covered if found.lower() in wanted else unknown
+            key = wanted.get(found.lower(), found)
+            if name not in target.setdefault(key, []):
+                target[key].append(name)
+    ordered = {i: covered[i] for i in ids if i in covered}
+    return Result(ordered, [i for i in ids if i not in covered], dict(sorted(unknown.items())))
+
+
+def _files(names: list[str], limit: int = 3) -> str:
+    extra = f" and {len(names) - limit} more" if len(names) > limit else ""
+    return printable(", ".join(names[:limit]) + extra)
+
+
+def render(result: Result) -> list[str]:
+    total = len(result.covered) + len(result.missing)
+    out = [f"trace: {len(result.covered)} of {total} requirements have tests"]
+    out += [f"  {printable(i)}  {_files(names)}" for i, names in result.covered.items()]
+    if result.missing:
+        out.append(f"missing: {printable(', '.join(result.missing))}")
+    out += [f"unknown in tests: {printable(i)} ({_files(names)})" for i, names in result.unknown.items()]
+    return out
+
+
+def decode(data: bytes) -> str:
+    if data.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
+        return data.decode("utf-16")
+    return data.decode("utf-8-sig")
+
+
+def _read(name: str) -> str:
+    if name != "-":
+        return decode(Path(name).read_bytes())
+    return decode(sys.stdin.buffer.read()) if hasattr(sys.stdin, "buffer") else sys.stdin.read()
+
+
+def cli(args) -> int:
+    utf8_stdout()
+    try:
+        ids = read_ids(_read(args.ids))
+    except OSError as e:
+        print(f"code-jig: {args.ids}: {e.strerror or e}", file=sys.stderr)
+        return 2
+    except ValueError as e:
+        print(f"code-jig: {args.ids}: {e}", file=sys.stderr)
+        return 2
+    if not ids:
+        print(f"code-jig: no requirement IDs in {args.ids}", file=sys.stderr)
+        return 2
+    try:
+        result = trace(Path(args.path).resolve(), ids, args.pattern)
+    except (ValueError, re.error) as e:
+        print(f"code-jig: --pattern: {e}", file=sys.stderr)
+        return 2
+    if args.json:
+        print(json.dumps({"covered": result.covered, "missing": result.missing, "unknown": result.unknown}, ensure_ascii=False, indent=1))
+    else:
+        for line in render(result):
+            print(line)
+    return 1 if result.missing or result.unknown else 0