lcode-cli 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

lcode/__init__.py

@@ -1,3 +1,3 @@
 """lcode — a local-first terminal coding agent powered by open-weight models via Ollama."""
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"

lcode/agent.py

@@ -154,6 +154,7 @@ class Agent:
         self.session_title = ""
         self.ctx_used = 0
         self.last_speed = 0.0
+        self.usage = {"requests": 0, "prompt_tokens": 0, "prompt_ns": 0, "output_tokens": 0, "output_ns": 0}
         self.messages: list[dict] = []
         self.reset()
 
@@ -390,6 +391,11 @@ class Agent:
             message["tool_calls"] = tool_calls
         self.messages.append(message)
         if final:
+            self.usage["requests"] += 1
+            self.usage["prompt_tokens"] += final.get("prompt_eval_count", 0)
+            self.usage["prompt_ns"] += final.get("prompt_eval_duration", 0)
+            self.usage["output_tokens"] += final.get("eval_count", 0)
+            self.usage["output_ns"] += final.get("eval_duration", 0)
             self.ctx_used = final.get("prompt_eval_count", 0) + final.get("eval_count", 0)
             seconds = final.get("eval_duration", 0) / 1e9
             self.last_speed = final.get("eval_count", 0) / seconds if seconds else 0.0

lcode/bench.py

@@ -0,0 +1,1082 @@
+"""lcode bench: score models on small, self-contained coding tasks on this machine.
+
+Each task runs in a fresh temporary folder in `yolo` mode without web access, and ends with an
+automatic check of the files or the answer. Some checks rerun the model's code on data it never
+saw, so hard-coding the expected output doesn't pass.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import io
+import json
+import random
+import re
+import shutil
+import signal
+import statistics
+import subprocess
+import sys
+import tempfile
+import time
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+from rich.console import Console
+from rich.markup import escape
+from rich.table import Table
+from rich.text import Text
+
+from lcode import __version__
+from lcode.agent import Agent, Settings
+from lcode.config import format_tokens
+from lcode.hardware import Hardware
+from lcode.ollama import Ollama, OllamaError
+
+SCHEMA_VERSION = 1
+DEFAULT_CONTEXT = 32768  # the same everywhere, so results compare across machines
+DEFAULT_TIMEOUT = 300  # seconds per task
+CHECK_TIMEOUT = 60
+PROBE_WORDS = 3000  # about 4,000 tokens
+PROBE_TEXT = (
+    "file function value error test module class return import list string number change build user "
+    "server request cache memory thread order parse check config option result folder branch commit"
+)
+
+
+class TaskTimeout(KeyboardInterrupt):
+    """Raised when a task runs out of time. The agent stops running commands just like on Ctrl+C."""
+
+
+@dataclass
+class Check:
+    passed: bool
+    detail: str
+
+
+@dataclass
+class Task:
+    id: str
+    title: str
+    prompt: str
+    files: dict[str, str]
+    check: Callable[[Path, str], Check]  # (task folder, the model's final answer) -> result
+
+
+# ----------------------------------------------------------------------------- helpers for checks
+
+
+def run_python(folder: Path, *args: str) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        [sys.executable, *args],
+        cwd=folder,
+        capture_output=True,
+        text=True,
+        stdin=subprocess.DEVNULL,
+        timeout=CHECK_TIMEOUT,
+    )
+
+
+def read(folder: Path, name: str) -> str | None:
+    try:
+        return (folder / name).read_text()
+    except (OSError, UnicodeDecodeError):
+        return None
+
+
+def last_line(text: str) -> str:
+    lines = [ln for ln in text.strip().splitlines() if ln.strip()]
+    return lines[-1][:200] if lines else ""
+
+
+def unittest_check(folder: Path, what: str) -> Check:
+    r = run_python(folder, "-m", "unittest", "-q")
+    if r.returncode == 0:
+        return Check(True, f"{what} pass")
+    return Check(False, f"{what} fail: {last_line(r.stderr) or last_line(r.stdout)}")
+
+
+# ----------------------------------------------------------------------------- 1. fix a bug
+
+CART = '''\
+"""A tiny shopping cart."""
+
+
+class Cart:
+    def __init__(self):
+        self.items = []
+
+    def add(self, name, price, quantity=1):
+        self.items.append((name, price, quantity))
+
+    def total(self, discount=0.0):
+        """Total price after an optional discount between 0 and 1."""
+        subtotal = sum(price for _, price, quantity in self.items)
+        return round(subtotal * (1 - discount), 2)
+'''
+
+CART_TESTS = """\
+import unittest
+
+from cart import Cart
+
+
+class CartTest(unittest.TestCase):
+    def test_total_counts_quantities(self):
+        cart = Cart()
+        cart.add("apple", 0.5, quantity=4)
+        cart.add("bread", 2.25)
+        self.assertEqual(cart.total(), 4.25)
+
+    def test_discount(self):
+        cart = Cart()
+        cart.add("coffee", 10.0, quantity=2)
+        self.assertEqual(cart.total(discount=0.1), 18.0)
+
+
+if __name__ == "__main__":
+    unittest.main()
+"""
+
+
+def check_fix_bug(folder: Path, answer: str) -> Check:
+    if read(folder, "test_cart.py") != CART_TESTS:
+        return Check(False, "test_cart.py was changed; the fix belongs in cart.py")
+    return unittest_check(folder, "the tests")
+
+
+# ----------------------------------------------------------------------------- 2. find code
+
+UPLOADER = {
+    "uploader/__init__.py": '"""Upload files to the storage service."""\n',
+    "uploader/config.py": '''\
+"""Settings for the uploader."""
+
+BASE_DELAY = 0.5  # seconds
+MAX_DELAY = 30.0
+MAX_ATTEMPTS = 6
+POLL_INTERVAL = 2.0
+CHUNK_SIZE = 4 * 1024 * 1024
+''',
+    "uploader/client.py": '''\
+"""Talking to the storage service."""
+
+import time
+
+from uploader.config import POLL_INTERVAL
+
+
+class StorageClient:
+    def __init__(self, session):
+        self.session = session
+
+    def put_chunk(self, url, data):
+        response = self.session.put(url, data=data)
+        response.raise_for_status()
+        return response
+
+    def wait_until_ready(self, job_id):
+        """Poll the service until a processing job has finished."""
+        while self.session.get(f"/jobs/{job_id}").json()["state"] != "done":
+            time.sleep(POLL_INTERVAL)
+''',
+    "uploader/backoff.py": '''\
+"""Spacing out attempts."""
+
+import random
+
+from uploader.config import BASE_DELAY, MAX_DELAY
+
+
+def jitter(delay):
+    """Spread attempts out so clients don't all hit the service at once."""
+    return delay * random.uniform(0.8, 1.2)
+
+
+def next_wait(attempt):
+    """Seconds to pause after `attempt` failed tries of the same chunk."""
+    delay = min(MAX_DELAY, BASE_DELAY * 2**attempt)
+    return jitter(delay)
+''',
+    "uploader/transfer.py": '''\
+"""Uploading a file in chunks."""
+
+import time
+
+from uploader.backoff import next_wait
+from uploader.config import CHUNK_SIZE, MAX_ATTEMPTS
+
+
+def chunks(path):
+    with open(path, "rb") as f:
+        while block := f.read(CHUNK_SIZE):
+            yield block
+
+
+def upload(client, url, path):
+    for index, block in enumerate(chunks(path)):
+        for attempt in range(MAX_ATTEMPTS):
+            try:
+                client.put_chunk(f"{url}?part={index}", block)
+                break
+            except OSError:
+                time.sleep(next_wait(attempt))
+        else:
+            raise RuntimeError(f"chunk {index} failed {MAX_ATTEMPTS} times")
+''',
+}
+
+
+def check_find_code(folder: Path, answer: str) -> Check:
+    lines = UPLOADER["uploader/backoff.py"].splitlines()
+    start = next(i for i, line in enumerate(lines, 1) if line.startswith("def next_wait"))
+    end = start + 3  # def, docstring and two lines of body
+    cited = [int(n) for n in re.findall(r"backoff\.py:(\d+)", answer)]
+    if "next_wait" not in answer:
+        named = re.findall(r"`([A-Za-z_]\w*)(?:\(\))?`", answer)
+        return Check(False, f"named {named[0]} instead of next_wait" if named else "didn't name next_wait")
+    if not cited:
+        return Check(False, "named next_wait but didn't cite backoff.py:<line>")
+    if not any(start <= n <= end for n in cited):
+        return Check(False, f"cited backoff.py:{cited[0]}, but next_wait is at lines {start}-{end}")
+    return Check(True, f"next_wait at backoff.py:{cited[0]}")
+
+
+# ----------------------------------------------------------------------------- 3. write a script
+
+NUMBERS = "12\n7\n\n3\n25\n8\n19\n"
+HIDDEN_NUMBERS = "5\n-2\n\n10\n4\n4\n"  # checked with data the model never saw
+
+
+def expected_stats(data: str) -> list[str]:
+    numbers = [int(line) for line in data.splitlines() if line.strip()]
+    return [
+        f"count: {len(numbers)}",
+        f"sum: {sum(numbers)}",
+        f"mean: {statistics.mean(numbers):.2f}",
+        f"median: {statistics.median(numbers):.2f}",
+    ]
+
+
+def check_write_script(folder: Path, answer: str) -> Check:
+    if not (folder / "stats.py").is_file():
+        return Check(False, "stats.py wasn't created")
+    for label, data in (("numbers.txt", NUMBERS), ("other data", HIDDEN_NUMBERS)):
+        (folder / "numbers.txt").write_text(data)
+        r = run_python(folder, "stats.py")
+        got = [line.strip() for line in r.stdout.strip().splitlines()]
+        want = expected_stats(data)
+        if r.returncode != 0:
+            return Check(False, f"stats.py failed on {label}: {last_line(r.stderr)}")
+        if got != want:
+            return Check(False, f"wrong output on {label}: {' | '.join(got)[:120]} (expected {' | '.join(want)})")
+    return Check(True, "correct output, also on data it hadn't seen")
+
+
+# ----------------------------------------------------------------------------- 4. rename across files
+
+SHOP = {
+    "shop/__init__.py": "",
+    "shop/users.py": '''\
+"""Users of the shop."""
+
+
+def get_user_name(user):
+    """The name to show for a user: their nickname if they have one."""
+    return user.get("nickname") or f"{user['first']} {user['last']}"
+''',
+    "shop/orders.py": """\
+from shop.users import get_user_name
+
+
+def order_summary(order):
+    return f"Order {order['id']} for {get_user_name(order['user'])}: {len(order['items'])} item(s)"
+""",
+    "shop/emails.py": """\
+from shop import users
+
+
+def greeting(user):
+    return f"Hello {users.get_user_name(user)},"
+""",
+    "test_shop.py": """\
+import unittest
+
+from shop.emails import greeting
+from shop.orders import order_summary
+from shop.users import get_user_name
+
+ADA = {"first": "Ada", "last": "Lovelace"}
+BOB = {"first": "Robert", "last": "Smith", "nickname": "Bob"}
+
+
+class ShopTest(unittest.TestCase):
+    def test_name(self):
+        self.assertEqual(get_user_name(ADA), "Ada Lovelace")
+        self.assertEqual(get_user_name(BOB), "Bob")
+
+    def test_callers(self):
+        self.assertEqual(greeting(BOB), "Hello Bob,")
+        self.assertEqual(order_summary({"id": 7, "user": ADA, "items": [1, 2]}), "Order 7 for Ada Lovelace: 2 item(s)")
+
+
+if __name__ == "__main__":
+    unittest.main()
+""",
+}
+
+HIDDEN_SHOP_TEST = """\
+import unittest
+
+from shop import users
+from shop.emails import greeting
+from shop.orders import order_summary
+
+ADA = {"first": "Ada", "last": "Lovelace"}
+BOB = {"first": "Robert", "last": "Smith", "nickname": "Bob"}
+
+
+class HiddenTest(unittest.TestCase):
+    def test_renamed(self):
+        self.assertFalse(hasattr(users, "get_user_name"))
+        self.assertEqual(users.display_name(ADA), "Ada Lovelace")
+        self.assertEqual(users.display_name(BOB), "Bob")
+        self.assertEqual(greeting(ADA), "Hello Ada Lovelace,")
+        self.assertEqual(order_summary({"id": 1, "user": BOB, "items": []}), "Order 1 for Bob: 0 item(s)")
+"""
+
+
+def check_rename(folder: Path, answer: str) -> Check:
+    left = sorted(
+        str(p.relative_to(folder))
+        for p in folder.rglob("*.py")
+        if "__pycache__" not in p.parts and "get_user_name" in (read(folder, str(p.relative_to(folder))) or "")
+    )
+    if left:
+        return Check(False, f"get_user_name is still used in {', '.join(left)}")
+    tests = folder / "test_shop.py"
+    if not tests.is_file() or "display_name" not in tests.read_text():
+        return Check(False, "test_shop.py wasn't updated")
+    (folder / "test_hidden_rename.py").write_text(HIDDEN_SHOP_TEST)
+    return unittest_check(folder, "its tests and extra checks")
+
+
+# ----------------------------------------------------------------------------- 5. precise edit in a long file
+
+
+def services_file() -> str:
+    lines = ['"""Connection settings for internal services."""', "", "SERVICES = {"]
+    for i in range(1, 151):
+        names = [f"service_{i:03d}"] + ([f"service_{i:03d}_replica"] if i % 29 == 0 else [])
+        for name in names:
+            lines += [
+                f'    "{name}": {{',
+                f'        "host": "{name.replace("_", "-")}.internal",',
+                f'        "port": {8000 + i},',
+                f'        "timeout": {60 if i % 10 == 0 else 30},',
+                f'        "retries": {5 if i % 7 == 0 else 3},',
+                "    },",
+            ]
+    lines += ["}", ""]
+    return "\n".join(lines)
+
+
+SERVICES = services_file()
+TARGET_SERVICE = "service_087"
+
+
+def check_precise_edit(folder: Path, answer: str) -> Check:
+    new = read(folder, "services.py")
+    if new is None:
+        return Check(False, "services.py is missing")
+    before, after = SERVICES.splitlines(), new.splitlines()
+    target = before.index(f'    "{TARGET_SERVICE}": {{') + 3  # its timeout line
+    expected = before.copy()
+    expected[target] = expected[target].replace("30", "45")
+    if after == expected:
+        return Check(True, f"changed only {TARGET_SERVICE}'s timeout")
+    if len(after) != len(before):
+        return Check(False, f"the file went from {len(before)} to {len(after)} lines")
+    changed = [i for i, (a, b) in enumerate(zip(before, after, strict=True)) if a != b]
+    if target not in changed:
+        return Check(False, f"{TARGET_SERVICE}'s timeout wasn't changed to 45")
+    return Check(False, f"changed {len(changed)} lines instead of 1")
+
+
+# ----------------------------------------------------------------------------- 6. recover from errors
+
+TOOL = '''\
+"""Build a ranked report from a data file.
+
+Usage: python tool.py --source DATA --out REPORT
+"""
+
+import argparse
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Build a ranked report from a data file.")
+    parser.add_argument("--source", help="data file with one 'name,score' per line")
+    parser.add_argument("--out", help="where to write the report")
+    parser.add_argument("--input", help=argparse.SUPPRESS)
+    args = parser.parse_args()
+    if args.input:
+        sys.exit("error: --input was removed in version 2; use --source instead")
+    if not args.source or not args.out:
+        parser.error("--source and --out are required")
+    rows = []
+    with open(args.source) as f:
+        for number, line in enumerate(f, 1):
+            if not line.strip():
+                continue
+            name, _, score = line.partition(",")
+            if not score.strip().isdigit():
+                sys.exit(f"error: {args.source} line {number}: score must be a whole number, got {score.strip()!r}")
+            rows.append((name.strip(), int(score)))
+    rows.sort(key=lambda row: (-row[1], row[0]))
+    with open(args.out, "w") as f:
+        for rank, (name, score) in enumerate(rows, 1):
+            f.write(f"{rank}. {name} {score}\\n")
+        f.write(f"total {sum(score for _, score in rows)}\\n")
+
+
+if __name__ == "__main__":
+    main()
+'''
+
+SCORES = "alice, 42\nbob, 17\ncarol, 42\ndave, n/a\nerin, 8\n"
+GOOD_ROWS = [("alice", 42), ("carol", 42), ("bob", 17), ("erin", 8)]
+
+
+def check_recover(folder: Path, answer: str) -> Check:
+    report = read(folder, "report.txt")
+    expected = "".join(f"{rank}. {name} {score}\n" for rank, (name, score) in enumerate(GOOD_ROWS, 1))
+    expected += f"total {sum(score for _, score in GOOD_ROWS)}\n"
+    if report is None:
+        return Check(False, "report.txt wasn't created")
+    data = read(folder, "data.txt") or ""
+    if "dave" in data or any(name not in data for name, _ in GOOD_ROWS):
+        return Check(False, "data.txt should lose only the invalid line")
+    if report != expected:
+        return Check(False, f"report.txt is wrong: {' | '.join(report.splitlines())[:120]}")
+    return Check(True, "used --source and dropped the invalid line")
+
+
+# ----------------------------------------------------------------------------- 7. fix to a written spec
+
+DURATIONS = '''\
+import re
+
+UNITS = {"h": 3600, "m": 60, "s": 1}
+
+
+def parse_duration(text):
+    """Parse a duration such as "1h30m", "45s" or "2h" into seconds.
+
+    - The units are h, m and s, in that order, each at most once: "1h0m5s" is fine, "30m1h" is not.
+    - Units are case-insensitive, and spaces between the parts are allowed: "1H 30m".
+    - A plain number means seconds: "90".
+    - Anything else raises ValueError, including an empty string and unknown units like "1x".
+    """
+    total = 0
+    for number, unit in re.findall(r"(\\d+)([hms])", text):
+        total = int(number) * UNITS[unit]
+    return total
+'''
+
+HIDDEN_DURATION_TEST = """\
+import unittest
+
+from durations import parse_duration
+
+
+class HiddenTest(unittest.TestCase):
+    def test_valid(self):
+        cases = {"1h30m": 5400, "45s": 45, "2h": 7200, "1h0m5s": 3605, "90m": 5400, "1H 30m": 5400,
+                 "90": 90, "0s": 0, "2h 5s": 7205, "1h 2m 3s": 3723}
+        for text, seconds in cases.items():
+            with self.subTest(text=text):
+                self.assertEqual(parse_duration(text), seconds)
+
+    def test_invalid(self):
+        for text in ["", "1x", "30m1h", "h", "1h1h", "abc", "5m 3m"]:
+            with self.subTest(text=text):
+                with self.assertRaises(ValueError):
+                    parse_duration(text)
+"""
+
+
+def check_spec_fix(folder: Path, answer: str) -> Check:
+    if not (folder / "durations.py").is_file():
+        return Check(False, "durations.py is missing")
+    (folder / "test_hidden_durations.py").write_text(HIDDEN_DURATION_TEST)
+    r = run_python(folder, "-m", "unittest", "-q", "test_hidden_durations")
+    if r.returncode == 0:
+        return Check(True, "follows the whole docstring")
+    failed = sorted(set(re.findall(r"\(text='([^']*)'\)", r.stderr)))
+    if failed:
+        return Check(False, "wrong for " + ", ".join(repr(t) for t in failed[:6]))
+    return Check(False, f"hidden tests fail: {last_line(r.stderr)}")
+
+
+# ----------------------------------------------------------------------------- 8. a feature across files
+
+TODO_APP = {
+    "storage.py": '''\
+"""Todos live in todos.json in the current folder."""
+
+import json
+from pathlib import Path
+
+PATH = Path("todos.json")
+
+
+def load():
+    if not PATH.exists():
+        return []
+    return json.loads(PATH.read_text())
+
+
+def save(todos):
+    PATH.write_text(json.dumps(todos, indent=2))
+''',
+    "todo.py": '''\
+"""A tiny todo list: python todo.py add TITLE | list | done ID"""
+
+import argparse
+
+import storage
+
+
+def add(title):
+    todos = storage.load()
+    next_id = max((t["id"] for t in todos), default=0) + 1
+    todos.append({"id": next_id, "title": title, "done": False})
+    storage.save(todos)
+    print(f"Added #{next_id}")
+
+
+def done(todo_id):
+    todos = storage.load()
+    for todo in todos:
+        if todo["id"] == todo_id:
+            todo["done"] = True
+            storage.save(todos)
+            print(f"Done #{todo_id}")
+            return
+    raise SystemExit(f"No todo #{todo_id}")
+
+
+def format_todo(todo):
+    return f"#{todo['id']} [{'x' if todo['done'] else ' '}] {todo['title']}"
+
+
+def list_todos():
+    for todo in storage.load():
+        print(format_todo(todo))
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description="A tiny todo list")
+    commands = parser.add_subparsers(dest="command", required=True)
+    add_command = commands.add_parser("add")
+    add_command.add_argument("title")
+    done_command = commands.add_parser("done")
+    done_command.add_argument("id", type=int)
+    commands.add_parser("list")
+    args = parser.parse_args(argv)
+    if args.command == "add":
+        add(args.title)
+    elif args.command == "done":
+        done(args.id)
+    else:
+        list_todos()
+
+
+if __name__ == "__main__":
+    main()
+''',
+    "test_todo.py": """\
+import contextlib
+import io
+import tempfile
+import unittest
+from pathlib import Path
+
+import storage
+import todo
+
+
+def run(*argv):
+    out = io.StringIO()
+    with contextlib.redirect_stdout(out):
+        todo.main(list(argv))
+    return out.getvalue().splitlines()
+
+
+class TodoTest(unittest.TestCase):
+    def setUp(self):
+        self.folder = tempfile.TemporaryDirectory()
+        storage.PATH = Path(self.folder.name) / "todos.json"
+
+    def tearDown(self):
+        self.folder.cleanup()
+
+    def test_add_and_list(self):
+        self.assertEqual(run("add", "buy milk"), ["Added #1"])
+        self.assertEqual(run("add", "call mom"), ["Added #2"])
+        self.assertEqual(run("list"), ["#1 [ ] buy milk", "#2 [ ] call mom"])
+
+    def test_done(self):
+        run("add", "buy milk")
+        self.assertEqual(run("done", "1"), ["Done #1"])
+        self.assertEqual(run("list"), ["#1 [x] buy milk"])
+""",
+}
+
+TODO_STEPS = [
+    (["add", "low thing", "--priority", "low"], ["Added #2"]),
+    (["add", "urgent", "--priority", "high"], ["Added #3"]),
+    (["add", "normal thing"], ["Added #4"]),
+    (["add", "urgent two", "--priority", "high"], ["Added #5"]),
+    (["done", "4"], ["Done #4"]),
+    (
+        ["list"],
+        ["#3 [ ] ! urgent", "#5 [ ] ! urgent two", "#1 [ ] old task", "#4 [x] normal thing", "#2 [ ] low thing"],
+    ),
+]
+
+
+def check_feature(folder: Path, answer: str) -> Check:
+    tests = unittest_check(folder, "its tests")
+    if not tests.passed:
+        return tests
+    trial = folder / "hidden-check"
+    trial.mkdir()
+    for source in folder.glob("*.py"):
+        shutil.copy(source, trial / source.name)
+    (trial / "todos.json").write_text('[{"id": 1, "title": "old task", "done": false}]')  # saved before the change
+    for argv, want in TODO_STEPS:
+        r = run_python(trial, "todo.py", *argv)
+        got = [line.rstrip() for line in r.stdout.strip().splitlines()]
+        if r.returncode != 0 or got != want:
+            shown = " | ".join(got)[:120] or last_line(r.stderr)
+            return Check(False, f"`todo.py {' '.join(argv)}` printed {shown!r}, expected {' | '.join(want)!r}")
+    r = run_python(trial, "todo.py", "add", "x", "--priority", "urgent")
+    if r.returncode == 0:
+        return Check(False, "accepted an unknown priority")
+    return Check(True, "priorities work, including old todos")
+
+
+# ----------------------------------------------------------------------------- the suite
+
+TASKS = [
+    Task(
+        "fix-bug",
+        "Fix a bug so the tests pass",
+        "The tests in test_cart.py fail. Find the bug, fix it, and run the tests.",
+        {"cart.py": CART, "test_cart.py": CART_TESTS},
+        check_fix_bug,
+    ),
+    Task(
+        "find-code",
+        "Answer with file:line",
+        "Which function computes how long to wait before retrying a failed upload in this project? Reply with "
+        "the function name and where it is defined, as path:line (for example `pkg/module.py:12`).",
+        UPLOADER,
+        check_find_code,
+    ),
+    Task(
+        "write-script",
+        "Write a script from a spec",
+        "Write a Python script stats.py that reads numbers.txt (one integer per line; ignore blank lines) and "
+        "prints exactly these four lines:\ncount: <how many numbers>\nsum: <their sum>\n"
+        "mean: <the mean, with 2 decimals>\nmedian: <the median, with 2 decimals>\nThen run it to check the output.",
+        {"numbers.txt": NUMBERS},
+        check_write_script,
+    ),
+    Task(
+        "rename",
+        "Rename a function across files",
+        "Rename the function get_user_name to display_name everywhere in this project: the definition, every "
+        "caller and the tests. Then run the tests with `python -m unittest`.",
+        SHOP,
+        check_rename,
+    ),
+    Task(
+        "precise-edit",
+        "One-line edit in a long file",
+        f"In services.py, change the timeout of {TARGET_SERVICE} from 30 to 45. Change nothing else.",
+        {"services.py": SERVICES},
+        check_precise_edit,
+    ),
+    Task(
+        "recover",
+        "Recover from failing commands",
+        "Build report.txt from data.txt by running: python tool.py --input data.txt --out report.txt\n"
+        "If a line in data.txt is invalid, delete that line from data.txt and run the tool again.",
+        {"tool.py": TOOL, "data.txt": SCORES},
+        check_recover,
+    ),
+    Task(
+        "spec-fix",
+        "Fix a function to match its spec",
+        "Users report that parse_duration in durations.py is wrong: parse_duration('1h30m') returns 1800 instead "
+        "of 5400. Fix it so it does everything its docstring says. Add tests for the cases in the docstring to "
+        "test_durations.py and run them.",
+        {"durations.py": DURATIONS},
+        check_spec_fix,
+    ),
+    Task(
+        "feature",
+        "Add a feature across files",
+        "Add priorities to this todo app. `python todo.py add TITLE --priority high|normal|low` sets one "
+        "(the default is normal). `list` shows high-priority todos first, then normal, then low, keeping the order "
+        'they were added within each priority, and marks high-priority todos with "!" after the checkbox, like '
+        "`#3 [ ] ! call the bank`. Todos saved before this change have no priority and count as normal. Update "
+        "the tests and run them.",
+        TODO_APP,
+        check_feature,
+    ),
+]
+
+
+def select_tasks(ids: str | None) -> list[Task]:
+    if not ids:
+        return list(TASKS)
+    wanted = [i.strip() for i in ids.split(",") if i.strip()]
+    known = {task.id: task for task in TASKS}
+    unknown = [i for i in wanted if i not in known]
+    if unknown:
+        raise ValueError(f"unknown task(s) {', '.join(unknown)}; choose from {', '.join(known)}")
+    return [known[i] for i in wanted]
+
+
+# ----------------------------------------------------------------------------- running
+
+
+@dataclass
+class TaskResult:
+    id: str
+    passed: bool
+    seconds: float
+    detail: str
+    steps: int = 0  # model responses
+    tool_errors: int = 0  # failed tool calls, including malformed ones
+    error: str | None = None  # why the run stopped early: a timeout or an Ollama error
+
+
+@dataclass
+class ModelRun:
+    model: str  # as given: a catalog key or Ollama tag
+    ollama_model: str
+    context: int = 0
+    num_batch: int | None = None
+    think: bool = True
+    load_seconds: float = 0.0
+    prompt_tps: float | None = None  # measured with a fresh prompt; see prompt_speed()
+    memory_gb: float | None = None
+    gpu_percent: int | None = None
+    usage: dict = field(default_factory=dict)
+    tasks: list[TaskResult] = field(default_factory=list)
+    error: str | None = None  # the model couldn't run at all
+    interrupted: bool = False
+
+    @property
+    def passed(self) -> int:
+        return sum(t.passed for t in self.tasks)
+
+    @property
+    def seconds(self) -> float:
+        return sum(t.seconds for t in self.tasks)
+
+    @property
+    def generation_tps(self) -> float | None:
+        ns = self.usage.get("output_ns", 0)
+        return round(self.usage["output_tokens"] / (ns / 1e9), 1) if ns else None
+
+    @property
+    def tool_errors(self) -> int:
+        return sum(t.tool_errors for t in self.tasks)
+
+    def to_json(self) -> dict:
+        data = asdict(self)
+        data.update(
+            passed=self.passed,
+            total=len(self.tasks),
+            seconds=round(self.seconds, 1),
+            generation_tps=self.generation_tps,
+            tool_errors=self.tool_errors,
+        )
+        return data
+
+
+@contextmanager
+def time_limit(seconds: float) -> Iterator[None]:
+    def expire(signum, frame):
+        raise TaskTimeout
+
+    previous = signal.signal(signal.SIGALRM, expire)
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+        signal.signal(signal.SIGALRM, previous)
+
+
+class Elapsed:
+    def __init__(self, label: str):
+        self.label, self.start = label, time.monotonic()
+
+    def __rich__(self) -> Text:
+        return Text(f"{self.label}  {time.monotonic() - self.start:.0f}s", style="dim")
+
+
+def count_tool_errors(messages: list[dict]) -> int:
+    failed = sum(1 for m in messages if m.get("role") == "tool" and m.get("content", "").startswith("Error:"))
+    malformed = sum(
+        1
+        for m in messages
+        if m.get("role") == "user" and m.get("content", "").startswith("[lcode] Your last tool call")
+    )
+    return failed + malformed
+
+
+def run_task(
+    ollama: Ollama, settings: Settings, task: Task, timeout: float, log: Console, keep: bool, usage: dict
+) -> TaskResult:
+    folder = Path(tempfile.mkdtemp(prefix=f"lcode-bench-{task.id}-"))
+    for rel, content in task.files.items():
+        (folder / rel).parent.mkdir(parents=True, exist_ok=True)
+        (folder / rel).write_text(content)
+    agent = Agent(ollama, settings, folder, console=log)
+    error = None
+    started = time.monotonic()
+    try:
+        with time_limit(timeout):
+            agent.run_turn(task.prompt)
+    except TaskTimeout:
+        error = f"timed out after {timeout:.0f}s"
+    except OllamaError as e:
+        error = str(e).splitlines()[0][:200]
+    seconds = time.monotonic() - started
+    for key, value in agent.usage.items():
+        usage[key] = usage.get(key, 0) + value
+    answer = next((m.get("content", "") for m in reversed(agent.messages) if m.get("role") == "assistant"), "")
+    try:
+        check = task.check(folder, answer)
+    except Exception as e:  # a check must never crash the benchmark
+        check = Check(False, f"check failed: {type(e).__name__}: {e}")
+    if keep:
+        (folder / "lcode-bench.log").write_text(log.file.getvalue() if isinstance(log.file, io.StringIO) else "")
+    else:
+        shutil.rmtree(folder, ignore_errors=True)
+    return TaskResult(
+        id=task.id,
+        passed=check.passed and error is None,
+        seconds=round(seconds, 1),
+        detail=error or check.detail,
+        steps=sum(1 for m in agent.messages if m.get("role") == "assistant"),
+        tool_errors=count_tool_errors(agent.messages),
+        error=error,
+    )
+
+
+def prompt_speed(ollama: Ollama, settings: Settings, options: dict) -> float | None:
+    """Prompt tokens per second for prompts the model hasn't seen: the faster of two readings.
+
+    Measured separately: the tasks' requests mostly repeat the conversation so far, which Ollama
+    serves from its prompt cache, and counting those tokens would overstate the speed. The first
+    reading after loading can be much slower while the model warms up, hence two.
+    """
+    readings = [_read_prompt(ollama, settings, options) for _ in range(2)]
+    return max((r for r in readings if r), default=None)
+
+
+def _read_prompt(ollama: Ollama, settings: Settings, options: dict) -> float | None:
+    rng = random.Random()
+    vocabulary = PROBE_TEXT.split()
+    words = " ".join(rng.choice(vocabulary) for _ in range(PROBE_WORDS))
+    payload = {
+        "model": settings.model,
+        "messages": [{"role": "user", "content": f"{rng.random()}\n{words}\n\nReply with OK."}],
+        "think": False,
+        "keep_alive": settings.keep_alive,
+        "options": {**options, "num_predict": 1},
+    }
+    final: dict = {}
+    try:
+        with time_limit(120):
+            for chunk in ollama.chat_stream(payload):
+                if chunk.get("done"):
+                    final = chunk
+    except (OllamaError, TaskTimeout):
+        return None
+    count, ns = final.get("prompt_eval_count", 0), final.get("prompt_eval_duration", 0)
+    return round(count / (ns / 1e9), 1) if ns and count >= 1000 else None
+
+
+def memory_use(ollama: Ollama, model: str) -> tuple[float | None, int | None]:
+    for entry in ollama.running():
+        if entry.get("name") in (model, f"{model}:latest") or entry.get("model") in (model, f"{model}:latest"):
+            size, vram = entry.get("size") or 0, entry.get("size_vram") or 0
+            if size:
+                return round(size / 1e9, 1), round(100 * vram / size)
+    return None, None
+
+
+def run_model(
+    ollama: Ollama,
+    name: str,
+    settings: Settings,
+    tasks: list[Task],
+    console: Console,
+    timeout: float = DEFAULT_TIMEOUT,
+    keep: bool = False,
+    verbose: bool = False,
+) -> ModelRun:
+    run = ModelRun(name, settings.model, settings.context, settings.num_batch, settings.think)
+    console.print(
+        f"\n[bold]{escape(settings.model)}[/] · {format_tokens(settings.context)} context · "
+        f"reasoning {'on' if settings.think else 'off'} · {len(tasks)} task{'' if len(tasks) == 1 else 's'}"
+    )
+    options = {"num_ctx": settings.context, **({"num_batch": settings.num_batch} if settings.num_batch else {})}
+    started = time.monotonic()
+    try:
+        with console.status(Elapsed("  Loading the model")):
+            ollama.load(settings.model, options, settings.keep_alive)
+    except OllamaError as e:
+        console.print(f"  [yellow]Loading failed ({escape(str(e).splitlines()[0][:150])}); the tasks will retry.[/]")
+    except KeyboardInterrupt:
+        run.interrupted = True
+        return run
+    run.load_seconds = round(time.monotonic() - started, 1)
+    run.memory_gb, run.gpu_percent = memory_use(ollama, settings.model)
+    try:
+        with console.status(Elapsed("  Measuring prompt speed")):
+            run.prompt_tps = prompt_speed(ollama, settings, options)
+    except KeyboardInterrupt:
+        run.interrupted = True
+        return run
+    where = f" · {run.memory_gb} GB, {run.gpu_percent}% on the GPU" if run.memory_gb else ""
+    reads = f" · reads prompts at {run.prompt_tps:,.0f} tok/s" if run.prompt_tps else ""
+    console.print(f"  [dim]Loaded in {run.load_seconds:.0f}s{where}{reads}[/]")
+    for task in tasks:
+        log = console if verbose else Console(file=io.StringIO(), width=120, force_terminal=False)
+        try:
+            if verbose:
+                console.rule(f"{task.id}: {task.title}")
+                result = run_task(ollama, settings, task, timeout, log, keep, run.usage)
+            else:
+                with console.status(Elapsed(f"  {task.id:<13} {task.title}")):
+                    result = run_task(ollama, settings, task, timeout, log, keep, run.usage)
+        except KeyboardInterrupt:
+            console.print("  [yellow]Stopped.[/]")
+            run.interrupted = True
+            break
+        run.tasks.append(result)
+        mark = "[green]✓[/]" if result.passed else "[red]✗[/]"
+        console.print(
+            f"  {mark} {task.id:<13} {task.title:<32} {result.seconds:>5.0f}s  [dim]{escape(result.detail)}[/]"
+        )
+    if run.memory_gb is None:
+        run.memory_gb, run.gpu_percent = memory_use(ollama, settings.model)
+    # lcode lowers the batch size or context if the GPU runs out of memory; report what was used.
+    run.context, run.num_batch, run.think = settings.context, settings.num_batch, settings.think
+    return run
+
+
+# ----------------------------------------------------------------------------- reporting
+
+
+def duration(seconds: float) -> str:
+    return f"{int(seconds // 60)}m {seconds % 60:02.0f}s" if seconds >= 60 else f"{seconds:.0f}s"
+
+
+def summary_rows(runs: list[ModelRun], tasks: list[Task]) -> list[tuple[str, list[str]]]:
+    rows = []
+    for task in tasks:
+        cells = []
+        for run in runs:
+            result = next((t for t in run.tasks if t.id == task.id), None)
+            cells.append("" if result is None else f"{'✓' if result.passed else '✗'} {duration(result.seconds)}")
+        rows.append((f"{task.id}: {task.title}", cells))
+
+    def cell(run: ModelRun, value: str) -> str:
+        return "—" if run.error else value
+
+    rows += [
+        ("Passed", [cell(r, f"{r.passed}/{len(r.tasks)}") for r in runs]),
+        ("Time", [cell(r, duration(r.seconds)) for r in runs]),
+        ("Generation", [cell(r, f"{r.generation_tps:.1f} tok/s" if r.generation_tps else "?") for r in runs]),
+        ("Prompt reading", [cell(r, f"{r.prompt_tps:,.0f} tok/s" if r.prompt_tps else "?") for r in runs]),
+        ("Tool-call errors", [cell(r, str(r.tool_errors)) for r in runs]),
+        (
+            "Memory",
+            [cell(r, f"{r.memory_gb} GB, {r.gpu_percent}% on GPU" if r.memory_gb else "?") for r in runs],
+        ),
+        ("Context", [cell(r, format_tokens(r.context)) for r in runs]),
+    ]
+    return rows
+
+
+def print_summary(console: Console, runs: list[ModelRun], tasks: list[Task]) -> None:
+    table = Table(title="lcode bench", title_justify="left", header_style="bold")
+    table.add_column("")
+    for run in runs:
+        table.add_column(escape(run.model), justify="right")
+    for label, cells in summary_rows(runs, tasks):
+        style = "bold" if label == "Passed" else ""
+        table.add_row(escape(label), *[escape(c) for c in cells], style=style)
+    console.print()
+    console.print(table)
+    for run in runs:
+        if run.error:
+            console.print(f"[yellow]{escape(run.model)}: {escape(run.error)}[/]")
+
+
+def report(runs: list[ModelRun], hardware: Hardware, ollama_version: str) -> dict:
+    """The JSON document written by --json (schema version 1)."""
+    return {
+        "schema": SCHEMA_VERSION,
+        "lcode": __version__,
+        "ollama": ollama_version,
+        "date": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
+        "hardware": {**asdict(hardware), "description": hardware.describe()},
+        "tasks": [{"id": t.id, "title": t.title} for t in TASKS],
+        "runs": [run.to_json() for run in runs],
+    }
+
+
+def markdown(runs: list[ModelRun], tasks: list[Task], hardware: Hardware, ollama_version: str) -> str:
+    """A table to paste into a model test report."""
+    date = dt.date.today().isoformat()
+    lines = [
+        f"**lcode bench** · lcode {__version__} · Ollama {ollama_version} · {hardware.describe()} · {date}",
+        "",
+        "| | " + " | ".join(f"`{r.model}`" for r in runs) + " |",
+        "|---|" + "---|" * len(runs),
+    ]
+    for label, cells in summary_rows(runs, tasks):
+        bold = label == "Passed"
+        label_md = f"**{label}**" if bold else label
+        lines.append(f"| {label_md} | " + " | ".join(f"**{c}**" if bold and c else c for c in cells) + " |")
+    return "\n".join(lines) + "\n"
+
+
+def write_json(path: Path, document: dict) -> None:
+    path.write_text(json.dumps(document, indent=2) + "\n")
+
+
+def print_tasks(console: Console) -> None:
+    table = Table(title="lcode bench tasks", title_justify="left", header_style="bold")
+    table.add_column("Task", style="cyan")
+    table.add_column("What it tests")
+    for task in TASKS:
+        table.add_row(task.id, task.title)
+    console.print(table)
+    console.print("[dim]Run a subset with --tasks, e.g. lcode bench --tasks fix-bug,rename[/]")

lcode/cli.py

@@ -6,6 +6,7 @@ import argparse
 import platform
 import shutil
 import sys
+import tempfile
 from pathlib import Path
 
 import requests
@@ -343,6 +344,79 @@ def cmd_config(args) -> None:
         fail(str(e))
 
 
+# ----------------------------------------------------------------------------- lcode bench
+
+
+def cmd_bench(args) -> None:
+    from lcode import bench
+
+    if args.list:
+        bench.print_tasks(console)
+        return
+    try:
+        cfg = config.load()
+        context = parse_context(args.context) if args.context else bench.DEFAULT_CONTEXT
+        tasks = bench.select_tasks(args.tasks)
+    except (ConfigError, ValueError) as e:
+        fail(str(e))
+    if args.timeout <= 0:
+        fail("--timeout must be a positive number of seconds")
+    ollama = Ollama(cfg["ollama_host"])
+    hw = detect()
+    version = check_ollama(ollama, hw)
+    console.print(f"[bold]lcode bench[/] · {hw.describe()} · Ollama {version}")
+    console.print("[dim]Each task runs in a new temporary folder with every permission granted and web access off.[/]")
+    names = args.models or [cfg["model"]]
+    others = [m.get("name", "?") for m in ollama.running()]
+    if others:
+        console.print(
+            f"[yellow]Already loaded in Ollama: {', '.join(others)}. Models that share the GPU run slower; "
+            "for comparable results, stop other sessions first.[/]"
+        )
+    runs = []
+    for i, name in enumerate(names):
+        try:
+            model, spec = resolve_model(ollama, name)
+        except NotInstalled as e:
+            console.print(f"\n[yellow]Skipping {name}: {e}[/]")
+            runs.append(bench.ModelRun(name, name, error=str(e)))
+            continue
+        ctx, note = choose_context(ollama, model, spec, context, hw)
+        if note:
+            console.print(f"[yellow]{model}, context {format_tokens(ctx)}: {note}[/]")
+        num_batch = cfg["num_batch"]
+        if num_batch is None and spec and spec.num_batch and model == spec.local_name:
+            num_batch = spec.num_batch
+        settings = Settings(
+            model=model,
+            context=ctx,
+            num_batch=num_batch,
+            keep_alive=cfg["keep_alive"],
+            think=cfg["think"] and not args.no_think,
+            permission_mode="yolo",
+            web="off",
+            checkpoints=False,
+        )
+        run = bench.run_model(ollama, name, settings, tasks, console, args.timeout, args.keep, args.verbose)
+        runs.append(run)
+        if run.interrupted:
+            break
+        if i < len(names) - 1:
+            try:
+                ollama.unload(model)  # so the next model gets all the GPU memory
+            except OllamaError:
+                pass
+    bench.print_summary(console, runs, tasks)
+    if args.json:
+        path = Path(args.json).expanduser()
+        bench.write_json(path, bench.report(runs, hw, version))
+        console.print(f"Results saved to {path}")
+    if args.markdown:
+        print("\n" + bench.markdown(runs, tasks, hw, version))
+    if args.keep:
+        console.print(f"[dim]Task folders are kept in {tempfile.gettempdir()} (lcode-bench-*).[/]")
+
+
 # ----------------------------------------------------------------------------- lcode (chat)
 
 
@@ -400,7 +474,7 @@ def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog="lcode",
         description="A local-first terminal coding agent powered by open-weight models via Ollama.",
-        epilog="Subcommands: lcode setup | models | doctor | config  (lcode <subcommand> --help). "
+        epilog="Subcommands: lcode setup | models | doctor | config | bench  (lcode <subcommand> --help). "
         "Docs: https://nasser1941.github.io/lcode/",
     )
     parser.add_argument("-p", "--prompt", help="run one request non-interactively and exit")
@@ -438,10 +512,25 @@ def build_subparsers() -> dict[str, argparse.ArgumentParser]:
     p.add_argument("key", nargs="?", choices=list(config.SETTINGS))
     p.add_argument("value", nargs="?")
     subs["config"] = p
+    p = argparse.ArgumentParser(
+        prog="lcode bench",
+        description="Score models on small coding tasks on this machine: pass rate, speed and memory.",
+    )
+    p.add_argument("models", nargs="*", help="catalog keys or Ollama tags (default: the configured model)")
+    p.add_argument("--context", "--ctx", dest="context", help="context window (default: 32k, the same everywhere)")
+    p.add_argument("--tasks", help="comma-separated task ids to run (default: all; see --list)")
+    p.add_argument("--timeout", type=float, default=300, help="seconds per task (default: 300)")
+    p.add_argument("--json", metavar="FILE", help="also save the results as JSON")
+    p.add_argument("--markdown", action="store_true", help="also print a Markdown table for a test report")
+    p.add_argument("--no-think", action="store_true", help="run without model reasoning")
+    p.add_argument("--keep", action="store_true", help="keep each task's folder and transcript for inspection")
+    p.add_argument("-v", "--verbose", action="store_true", help="show the model working, like a normal session")
+    p.add_argument("--list", action="store_true", help="list the tasks and exit")
+    subs["bench"] = p
     return subs
 
 
-COMMANDS = {"setup": cmd_setup, "models": cmd_models, "doctor": cmd_doctor, "config": cmd_config}
+COMMANDS = {"setup": cmd_setup, "models": cmd_models, "doctor": cmd_doctor, "config": cmd_config, "bench": cmd_bench}
 
 
 def main(argv: list[str] | None = None) -> None:

lcode/ollama.py

@@ -147,6 +147,10 @@ class Ollama:
                         raise OllamaError(f"Ollama error: {chunk['error']}")
                     yield chunk
 
+    def unload(self, model: str) -> None:
+        """Free the memory a model is using."""
+        self._post("/api/generate", {"model": model, "keep_alive": 0}, timeout=60)
+
     def load(self, model: str, options: dict, keep_alive: str) -> None:
         """Load a model into memory without generating anything."""
         self._post(

{lcode_cli-0.3.0.dist-info → lcode_cli-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: lcode-cli
-Version: 0.3.0
+Version: 0.3.1
 Summary: A local-first terminal coding agent powered by open-weight models on your own GPU or Mac, via Ollama.
 Project-URL: Homepage, https://nasser1941.github.io/lcode/
 Project-URL: Documentation, https://nasser1941.github.io/lcode/
@@ -73,7 +73,8 @@ through [Ollama](https://ollama.com), and when it needs current information it c
   plans multi-step tasks with a visible todo list.
 - **Long context.** 256K tokens with the default model (1M with Nemotron). You choose the window.
 - **Hardware-aware.** Detects your NVIDIA GPU or Apple Silicon Mac and picks the best model and the
-  largest context that fits: `lcode setup` does it in one step.
+  largest context that fits: `lcode setup` does it in one step. `lcode bench` compares models on
+  real coding tasks on your machine.
 - **Safe by default.** Every edit is shown as a diff and every command that isn't read-only needs
   your approval. `auto-edit` and `yolo` modes when you want speed.
 - **Undo.** lcode saves a checkpoint before the model changes files; `/undo` takes back the last
@@ -125,6 +126,7 @@ lcode
 | `lcode -c` / `lcode --resume` | continue the last session here / pick a saved session from a list |
 | `lcode --model qwen3.5-9b --context 128k` | pick a model and context window for this session |
 | `lcode models` / `lcode doctor` | what fits this machine / check the installation |
+| `lcode bench qwen3.6-35b qwen3.5-9b` | compare models on small coding tasks on this machine |
 | `/undo`, `/rewind` | take back the last request's file changes, or go back further |
 | `/rename`, `/resume` | name the current session, resume a saved one |
 | `/model`, `/context`, `/compact`, `/help` | switch model, resize the context window (pick from a list), summarize, list commands |
@@ -170,7 +172,6 @@ fills up. Details: [How it works](https://nasser1941.github.io/lcode/how-it-work
 
 See the pinned [Roadmap issue](https://github.com/nasser1941/lcode/issues/10). Next up:
 
-- [`lcode bench`: score models on your own hardware](https://github.com/nasser1941/lcode/issues/5)
 - [MCP tool servers](https://github.com/nasser1941/lcode/issues/6)
 - [Homebrew formula](https://github.com/nasser1941/lcode/issues/8) and an [optional sandbox for shell commands](https://github.com/nasser1941/lcode/issues/9)
 - More models in the catalog, and test reports from your hardware ([report one](https://github.com/nasser1941/lcode/issues/new?template=model_request.yml))

{lcode_cli-0.3.0.dist-info → lcode_cli-0.3.1.dist-info}/RECORD

@@ -1,22 +1,23 @@
-lcode/__init__.py,sha256=6o9drjzBZ6xhvmNDRH_G_fyZUs1QniLoDVSjRcEmymU,117
+lcode/__init__.py,sha256=7co3jkJP_ri78Ap0JVyQJkT8sWh6fKXdJUCuT5A5Ab4,117
 lcode/__main__.py,sha256=s5N5pryX2FAu93GGZ3FxUrIXHWGzxz66lKUoaV_h8Ws,35
-lcode/agent.py,sha256=NtLV_QI75Ixvs-6Ig39qhMb2kPuIkK4z1qMclyBb3sQ,25335
+lcode/agent.py,sha256=VYYGmkQi8CnP7mXgA2n_tjXnYn44uwK00j4Ow7vGKmg,25776
+lcode/bench.py,sha256=72TSBit063ERFlzgsq2xuCSzWaCLGoZKhB3G_E5HlBM,37751
 lcode/catalog.py,sha256=pZ1azX9OH2r4JE2c6E5OfwywpgS39-W2oXzoCHJhmtM,4141
 lcode/checkpoints.py,sha256=eqPtmPe44-aweWqc2H21ROogwzIt3NpvcUobsaCHoGY,18354
-lcode/cli.py,sha256=rU5XYsYmnUU-N9N1GX_s0oWSwgqYM-8k_TEKR0tlzvs,20143
+lcode/cli.py,sha256=h830l_4BTgQCZULcKbMvKMV57v5g6Va0livr83vAF1E,24295
 lcode/config.py,sha256=xkD7L5gSrHbgnXt7C8SYktNXO62BqNIXtuGyCY-y4FY,6002
 lcode/hardware.py,sha256=2VbknZCxTZJwJUD8YWdQ7ryEQo_LcjtoOBOHOU9Y0fw,3085
 lcode/limits.py,sha256=oJzVQoMyTZ0_Fkpu5t3qxrf9v7Hv9S8U8BRQR5M1aKA,1237
 lcode/models.toml,sha256=lt2G64zzo_Ryxv2ISRSNZoVD_JqCqYqsM01n8WoXGs8,4098
-lcode/ollama.py,sha256=fELwqw7xqLH9TM7XvdI1pxqYZFeIRJBv-VSz9REdvwA,6516
+lcode/ollama.py,sha256=-EB_8DKh87larG7iV0neyndPRom1Hxh4y8AAfI1Ebq8,6690
 lcode/permissions.py,sha256=wJKwhJQx44AzjOFZCWwM4M2oUlPFUNYI2cIVfoBGGgc,3898
 lcode/render.py,sha256=gMvf4W3qkLUJ4Vot3az_hP_zRDAItJC9pP-68dHFkfA,1375
 lcode/repl.py,sha256=F5DsLYya5UmJt2LvJf-N_3CdBKxiK8sr7B0zNSS7gKI,26525
 lcode/sessions.py,sha256=wOR-J0EHgJxLSPboKPHnvBcp5VTUYTj3tZWgpsa6-K4,4013
 lcode/tools.py,sha256=0JFM-owT3K243tvMhHlcfWKSMyHSa44GY4A2GebKU0k,25935
 lcode/web.py,sha256=vyxKbSRJTl1Lj3hj7Yd_NwTgbv0Xi6dup2YfUXjqp-Y,14579
-lcode_cli-0.3.0.dist-info/METADATA,sha256=o6_TdM5_SW2xs3j5SyjtNE6iIfagy9ivcBS29xplJ_c,9423
-lcode_cli-0.3.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-lcode_cli-0.3.0.dist-info/entry_points.txt,sha256=pZ_uDQ98UzVAHUQeXn5HeyQWKGSH3r3T5aPHuSB89eI,41
-lcode_cli-0.3.0.dist-info/licenses/LICENSE,sha256=JWjmoL7nNIC1XUa-dZm2hwLUgjUQMFo6k8APb0idNzs,1074
-lcode_cli-0.3.0.dist-info/RECORD,,
+lcode_cli-0.3.1.dist-info/METADATA,sha256=8sici8niZzAGQwTLMDDsOckVIbZmYB3lXZ8Cqpn7F2o,9490
+lcode_cli-0.3.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+lcode_cli-0.3.1.dist-info/entry_points.txt,sha256=pZ_uDQ98UzVAHUQeXn5HeyQWKGSH3r3T5aPHuSB89eI,41
+lcode_cli-0.3.1.dist-info/licenses/LICENSE,sha256=JWjmoL7nNIC1XUa-dZm2hwLUgjUQMFo6k8APb0idNzs,1074
+lcode_cli-0.3.1.dist-info/RECORD,,