meshapi-code 0.4.4__py3-none-any.whl → 0.4.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meshapi/__init__.py +1 -1
- meshapi/cli.py +24 -2
- meshapi/client.py +70 -38
- meshapi/commands.py +37 -0
- meshapi/config.py +3 -0
- meshapi/optimize.py +213 -0
- meshapi/tools.py +7 -2
- {meshapi_code-0.4.4.dist-info → meshapi_code-0.4.5.dist-info}/METADATA +39 -1
- meshapi_code-0.4.5.dist-info/RECORD +21 -0
- {meshapi_code-0.4.4.dist-info → meshapi_code-0.4.5.dist-info}/WHEEL +1 -1
- meshapi_code-0.4.4.dist-info/RECORD +0 -20
- {meshapi_code-0.4.4.dist-info → meshapi_code-0.4.5.dist-info}/entry_points.txt +0 -0
- {meshapi_code-0.4.4.dist-info → meshapi_code-0.4.5.dist-info}/licenses/LICENSE +0 -0
- {meshapi_code-0.4.4.dist-info → meshapi_code-0.4.5.dist-info}/licenses/NOTICE +0 -0
meshapi/__init__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.4.
|
|
1
|
+
__version__ = "0.4.5"
|
meshapi/cli.py
CHANGED
|
@@ -409,7 +409,12 @@ def _port_open(port: int, host: str = "127.0.0.1") -> bool:
|
|
|
409
409
|
def _kill_server(pid: int) -> None:
|
|
410
410
|
"""SIGTERM the entire process group of a tracked server (best-effort)."""
|
|
411
411
|
try:
|
|
412
|
-
os.killpg
|
|
412
|
+
# os.killpg/os.getpgid are POSIX-only. On Windows there are no process
|
|
413
|
+
# groups (start_new_session is a no-op), so kill the single pid.
|
|
414
|
+
if hasattr(os, "killpg") and hasattr(os, "getpgid"):
|
|
415
|
+
os.killpg(os.getpgid(pid), signal.SIGTERM)
|
|
416
|
+
else:
|
|
417
|
+
os.kill(pid, signal.SIGTERM) # Windows: TerminateProcess, single pid
|
|
413
418
|
except (ProcessLookupError, PermissionError, OSError):
|
|
414
419
|
pass
|
|
415
420
|
|
|
@@ -874,7 +879,12 @@ def main() -> None:
|
|
|
874
879
|
signal.signal(signum, signal.SIG_DFL)
|
|
875
880
|
os.kill(os.getpid(), signum)
|
|
876
881
|
|
|
877
|
-
|
|
882
|
+
# SIGHUP is POSIX-only — referencing signal.SIGHUP on Windows raises
|
|
883
|
+
# AttributeError, so build the list conditionally instead of unconditionally.
|
|
884
|
+
_signals = [signal.SIGTERM]
|
|
885
|
+
if hasattr(signal, "SIGHUP"):
|
|
886
|
+
_signals.append(signal.SIGHUP)
|
|
887
|
+
for _sig in _signals:
|
|
878
888
|
try:
|
|
879
889
|
signal.signal(_sig, _signal_shutdown)
|
|
880
890
|
except (ValueError, OSError):
|
|
@@ -990,6 +1000,7 @@ def main() -> None:
|
|
|
990
1000
|
agg_cost = 0.0
|
|
991
1001
|
last_model = state["cfg"]["model"]
|
|
992
1002
|
last_usage: dict = {}
|
|
1003
|
+
last_optimize_plan = {}
|
|
993
1004
|
last_elapsed = 0.0
|
|
994
1005
|
try:
|
|
995
1006
|
# While-loop so the cap can be promoted dynamically the moment the
|
|
@@ -1057,6 +1068,7 @@ def main() -> None:
|
|
|
1057
1068
|
last_model = meta.get("model") or last_model
|
|
1058
1069
|
last_usage = meta.get("usage") or last_usage
|
|
1059
1070
|
last_elapsed += meta.get("elapsed", 0.0)
|
|
1071
|
+
last_optimize_plan = meta.get("optimize_plan") or last_optimize_plan
|
|
1060
1072
|
|
|
1061
1073
|
tool_calls = meta.get("tool_calls") or []
|
|
1062
1074
|
if not tool_calls:
|
|
@@ -1090,6 +1102,16 @@ def main() -> None:
|
|
|
1090
1102
|
f"[dim]{last_model} • {prompt_t}→{completion_t} tok • {cost_str} • "
|
|
1091
1103
|
f"session {fmt_usd(state['session_cost'])} • {last_elapsed:.1f}s[/dim]"
|
|
1092
1104
|
)
|
|
1105
|
+
if last_optimize_plan:
|
|
1106
|
+
if last_optimize_plan.get("degraded"):
|
|
1107
|
+
console.print(
|
|
1108
|
+
f"[yellow]⚡ optimize beta: {last_optimize_plan['degraded']}[/yellow]"
|
|
1109
|
+
)
|
|
1110
|
+
else:
|
|
1111
|
+
from .optimize import savings_line
|
|
1112
|
+
line = savings_line(last_optimize_plan, last_usage)
|
|
1113
|
+
if line:
|
|
1114
|
+
console.print(f"[dim]{line}[/dim]")
|
|
1093
1115
|
except KeyboardInterrupt:
|
|
1094
1116
|
console.rule(style="dim yellow", characters="─")
|
|
1095
1117
|
console.print("[yellow]aborted by user — returning to prompt[/yellow]")
|
meshapi/client.py
CHANGED
|
@@ -4,6 +4,8 @@ from typing import Iterable, Optional
|
|
|
4
4
|
|
|
5
5
|
import httpx
|
|
6
6
|
|
|
7
|
+
from .optimize import prepare
|
|
8
|
+
|
|
7
9
|
|
|
8
10
|
def stream_chat(
|
|
9
11
|
messages: list,
|
|
@@ -16,6 +18,12 @@ def stream_chat(
|
|
|
16
18
|
- `cost` arrives in the final SSE chunk alongside `usage`.
|
|
17
19
|
- `tool_calls` arrive as deltas indexed by position; we accumulate them
|
|
18
20
|
and surface as the meta dict's `tool_calls` field.
|
|
21
|
+
|
|
22
|
+
When the `optimize` dial is set (BETA), the request is rewritten by the
|
|
23
|
+
phase 1 lever stack in optimize.py before sending, and the plan rides on
|
|
24
|
+
the final meta dict as `optimize_plan`. If the gateway rejects the
|
|
25
|
+
optimized request, we retry the raw request once, so the beta can never
|
|
26
|
+
be the reason a turn fails.
|
|
19
27
|
"""
|
|
20
28
|
url = f"{cfg['base_url']}/chat/completions"
|
|
21
29
|
headers = {
|
|
@@ -33,57 +41,81 @@ def stream_chat(
|
|
|
33
41
|
payload["tools"] = tools
|
|
34
42
|
payload["tool_choice"] = "auto"
|
|
35
43
|
|
|
44
|
+
plan: dict = {}
|
|
45
|
+
attempts = [payload]
|
|
46
|
+
dial = float(cfg.get("optimize") or 0)
|
|
47
|
+
if dial > 0:
|
|
48
|
+
opt_messages, extra, plan = prepare(
|
|
49
|
+
messages, cfg["model"], dial, has_tools=bool(tools)
|
|
50
|
+
)
|
|
51
|
+
if plan.get("levers_applied"):
|
|
52
|
+
optimized = {**payload, **extra, "messages": opt_messages}
|
|
53
|
+
attempts = [optimized, payload] # raw payload is the fallback
|
|
54
|
+
|
|
36
55
|
last_meta: dict = {}
|
|
37
56
|
last_model: str = ""
|
|
38
57
|
tool_calls_accum: dict = {} # index -> {id, name, arguments}
|
|
39
58
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
59
|
+
for attempt_index, body in enumerate(attempts):
|
|
60
|
+
is_last_attempt = attempt_index == len(attempts) - 1
|
|
61
|
+
with httpx.stream("POST", url, json=body, headers=headers, timeout=120) as r:
|
|
62
|
+
if r.status_code >= 400:
|
|
63
|
+
r.read() # so e.response.text works in the caller
|
|
64
|
+
if not is_last_attempt:
|
|
65
|
+
# Optimized request rejected; degrade to the raw request.
|
|
66
|
+
plan = {
|
|
67
|
+
"dial": dial,
|
|
68
|
+
"levers_applied": [],
|
|
69
|
+
"degraded": f"gateway returned {r.status_code}, sent raw request",
|
|
70
|
+
}
|
|
71
|
+
continue
|
|
72
|
+
r.raise_for_status()
|
|
73
|
+
for line in r.iter_lines():
|
|
74
|
+
if not line or not line.startswith("data: "):
|
|
75
|
+
continue
|
|
76
|
+
data = line[6:]
|
|
77
|
+
if data.strip() == "[DONE]":
|
|
78
|
+
break
|
|
79
|
+
try:
|
|
80
|
+
obj = json.loads(data)
|
|
81
|
+
except json.JSONDecodeError:
|
|
82
|
+
continue
|
|
54
83
|
|
|
55
|
-
|
|
56
|
-
|
|
84
|
+
if obj.get("model"):
|
|
85
|
+
last_model = obj["model"]
|
|
57
86
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
87
|
+
choices = obj.get("choices") or []
|
|
88
|
+
if choices:
|
|
89
|
+
delta = choices[0].get("delta", {})
|
|
61
90
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
91
|
+
content = delta.get("content")
|
|
92
|
+
if content:
|
|
93
|
+
yield content
|
|
65
94
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
95
|
+
for tc in delta.get("tool_calls") or []:
|
|
96
|
+
idx = tc.get("index", 0)
|
|
97
|
+
bucket = tool_calls_accum.setdefault(
|
|
98
|
+
idx, {"id": "", "name": "", "arguments": ""}
|
|
99
|
+
)
|
|
100
|
+
if tc.get("id"):
|
|
101
|
+
bucket["id"] = tc["id"]
|
|
102
|
+
fn = tc.get("function") or {}
|
|
103
|
+
if fn.get("name"):
|
|
104
|
+
bucket["name"] = fn["name"]
|
|
105
|
+
if fn.get("arguments"):
|
|
106
|
+
bucket["arguments"] += fn["arguments"]
|
|
78
107
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
108
|
+
usage = obj.get("usage")
|
|
109
|
+
cost = obj.get("cost")
|
|
110
|
+
if usage or cost:
|
|
111
|
+
last_meta = {"usage": usage, "cost": cost}
|
|
112
|
+
break # this attempt streamed successfully
|
|
83
113
|
|
|
84
114
|
if last_model:
|
|
85
115
|
last_meta["model"] = last_model
|
|
86
116
|
if tool_calls_accum:
|
|
87
117
|
last_meta["tool_calls"] = [tool_calls_accum[i] for i in sorted(tool_calls_accum)]
|
|
118
|
+
if plan:
|
|
119
|
+
last_meta["optimize_plan"] = plan
|
|
88
120
|
if last_meta:
|
|
89
121
|
yield last_meta
|
meshapi/commands.py
CHANGED
|
@@ -115,6 +115,42 @@ def handle_command(cmd: str, state: dict) -> bool:
|
|
|
115
115
|
elif name == "/cost":
|
|
116
116
|
console.print(f"[dim]Session spend: {fmt_usd(state.get('session_cost', 0))}[/dim]")
|
|
117
117
|
|
|
118
|
+
elif name == "/optimize":
|
|
119
|
+
# BETA: Mesh Optimize dial. 0 = off (full bypass), up to 0.95.
|
|
120
|
+
if not arg:
|
|
121
|
+
cur = float(state["cfg"].get("optimize") or 0)
|
|
122
|
+
label = f"{cur}" if cur > 0 else "off"
|
|
123
|
+
console.print(
|
|
124
|
+
f"[dim]optimize (beta): {label}\n"
|
|
125
|
+
"usage: /optimize <0 to 0.95> e.g. /optimize 0.3\n"
|
|
126
|
+
" /optimize off\n"
|
|
127
|
+
"0+ injects prompt cache breakpoints and max_tokens defaults; "
|
|
128
|
+
"0.2+ also prunes consumed tool results from old turns. "
|
|
129
|
+
"Savings appear in the status line after each turn. This is a "
|
|
130
|
+
"beta feature; set /optimize off to bypass entirely.[/dim]"
|
|
131
|
+
)
|
|
132
|
+
else:
|
|
133
|
+
raw = arg.strip().lower()
|
|
134
|
+
try:
|
|
135
|
+
value = 0.0 if raw == "off" else float(raw)
|
|
136
|
+
except ValueError:
|
|
137
|
+
console.print("[red]Not a number. Use 0 to 0.95, or 'off'.[/red]")
|
|
138
|
+
else:
|
|
139
|
+
if not 0 <= value <= 0.95:
|
|
140
|
+
console.print("[red]Dial range is 0 to 0.95.[/red]")
|
|
141
|
+
else:
|
|
142
|
+
state["cfg"]["optimize"] = value
|
|
143
|
+
save_config(state["cfg"])
|
|
144
|
+
if value > 0:
|
|
145
|
+
console.print(
|
|
146
|
+
f"[dim]optimize (beta) set to {value}. Levers: cache "
|
|
147
|
+
"injection, max_tokens defaults"
|
|
148
|
+
+ (", tool result pruning" if value >= 0.2 else "")
|
|
149
|
+
+ ". /optimize off to disable.[/dim]"
|
|
150
|
+
)
|
|
151
|
+
else:
|
|
152
|
+
console.print("[dim]optimize off. Requests pass through untouched.[/dim]")
|
|
153
|
+
|
|
118
154
|
elif name == "/mode":
|
|
119
155
|
if not arg:
|
|
120
156
|
cur = state.get("mode", Mode.DEFAULT)
|
|
@@ -138,6 +174,7 @@ def handle_command(cmd: str, state: dict) -> bool:
|
|
|
138
174
|
"/clear-attach drop any queued image attachments\n"
|
|
139
175
|
"/system <txt> set system prompt\n"
|
|
140
176
|
"/cost show session spend\n"
|
|
177
|
+
"/optimize <dial> token savings, beta: 0 off, up to 0.95\n"
|
|
141
178
|
"/help show this\n\n"
|
|
142
179
|
"[dim]Image paths in a prompt auto-attach: drop /path/img.png in your\n"
|
|
143
180
|
"input and it's sent as a base64 image part. Wrap in backticks to keep\n"
|
meshapi/config.py
CHANGED
|
@@ -18,6 +18,9 @@ DEFAULT_CONFIG = {
|
|
|
18
18
|
"model": "anthropic/claude-sonnet-4.5",
|
|
19
19
|
"system": "You are a helpful coding assistant. Be concise.",
|
|
20
20
|
"route": None,
|
|
21
|
+
# Mesh Optimize dial (BETA). 0 = off. 0 to 0.95: how aggressively to
|
|
22
|
+
# cut token spend. See /optimize in the REPL and README for details.
|
|
23
|
+
"optimize": 0.0,
|
|
21
24
|
}
|
|
22
25
|
|
|
23
26
|
_DIR_MODE = stat.S_IRWXU # 0700
|
meshapi/optimize.py
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
"""Mesh Optimize (BETA) — gateway-level token savings, applied client-side.
|
|
2
|
+
|
|
3
|
+
BETA FEATURE. Off by default. Enable with `/optimize 0.3` or the `optimize`
|
|
4
|
+
config key. Behavior, savings math, and the lever stack may change between
|
|
5
|
+
releases. Set `/optimize 0` to bypass everything.
|
|
6
|
+
|
|
7
|
+
Python port of the phase 1 levers from the mesh-optimize reference
|
|
8
|
+
implementation (https://github.com/raushan-aifiesta/mesh-optimize):
|
|
9
|
+
|
|
10
|
+
dial 0+ cache_control breakpoint injection on stable prefixes,
|
|
11
|
+
max_tokens defaults per task class
|
|
12
|
+
dial 0.2+ pruning of tool results the model already consumed
|
|
13
|
+
|
|
14
|
+
Hard rules carried over from the reference implementation:
|
|
15
|
+
- deterministic: same input always produces the same output
|
|
16
|
+
- never touches content inside a client-set cache breakpoint
|
|
17
|
+
- savings are reported honestly or not at all
|
|
18
|
+
- the original message list is never mutated
|
|
19
|
+
"""
|
|
20
|
+
import copy
|
|
21
|
+
import hashlib
|
|
22
|
+
import json
|
|
23
|
+
import re
|
|
24
|
+
|
|
25
|
+
# tokens, chars/4 estimate. Below the per-model minimum a cache_control
|
|
26
|
+
# marker silently does nothing, so we do not inject one.
|
|
27
|
+
_CACHE_MINIMUMS = [
|
|
28
|
+
(re.compile(r"fable"), 2048),
|
|
29
|
+
(re.compile(r"sonnet-4-6"), 2048),
|
|
30
|
+
(re.compile(r"opus"), 4096),
|
|
31
|
+
(re.compile(r"haiku-4-5"), 4096),
|
|
32
|
+
]
|
|
33
|
+
_DEFAULT_CACHE_MINIMUM = 2048
|
|
34
|
+
|
|
35
|
+
_KEEP_RECENT_MESSAGES = 4
|
|
36
|
+
_TRUNCATE_TO_CHARS = 400
|
|
37
|
+
|
|
38
|
+
_CODE_RE = re.compile(r"```|(?:\bfunction\b|\bclass\b|\bimport\b|\bdef\b)\s")
|
|
39
|
+
_COMPLEX_RE = re.compile(
|
|
40
|
+
r"\b(refactor|implement|debug|architect|migrate|optimi[sz]e|analy[sz]e)\b",
|
|
41
|
+
re.IGNORECASE,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
_MAX_TOKENS_DEFAULTS = {
|
|
45
|
+
"routine": 1024,
|
|
46
|
+
"standard": 1024,
|
|
47
|
+
"complex": 4096,
|
|
48
|
+
"agentic": 4096,
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def normalize_model(model: str) -> str:
|
|
53
|
+
"""anthropic/claude-opus-4.8 -> claude-opus-4-8 (bare, dashed)."""
|
|
54
|
+
bare = (model or "").lower().rsplit("/", 1)[-1]
|
|
55
|
+
return bare.replace(".", "-")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _cache_minimum(model: str) -> int:
|
|
59
|
+
bare = normalize_model(model)
|
|
60
|
+
for pattern, minimum in _CACHE_MINIMUMS:
|
|
61
|
+
if pattern.search(bare):
|
|
62
|
+
return minimum
|
|
63
|
+
return _DEFAULT_CACHE_MINIMUM
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _est_tokens(value) -> int:
|
|
67
|
+
if value is None:
|
|
68
|
+
return 0
|
|
69
|
+
text = value if isinstance(value, str) else json.dumps(value, sort_keys=True)
|
|
70
|
+
return -(-len(text) // 4) # ceil division
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _msg_tokens(message: dict) -> int:
|
|
74
|
+
return _est_tokens(message.get("content")) + 4
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _classify(messages: list, has_tools: bool) -> str:
|
|
78
|
+
depth = len(messages)
|
|
79
|
+
sample = ""
|
|
80
|
+
for message in messages[-6:]:
|
|
81
|
+
content = message.get("content")
|
|
82
|
+
sample += (content if isinstance(content, str) else json.dumps(content)) + "\n"
|
|
83
|
+
if has_tools and depth > 6:
|
|
84
|
+
return "agentic"
|
|
85
|
+
if has_tools:
|
|
86
|
+
return "complex"
|
|
87
|
+
if _CODE_RE.search(sample) or _COMPLEX_RE.search(sample):
|
|
88
|
+
return "complex"
|
|
89
|
+
if sum(_msg_tokens(m) for m in messages[-6:]) < 150 and depth <= 4:
|
|
90
|
+
return "routine"
|
|
91
|
+
return "standard"
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _has_client_breakpoints(messages: list) -> bool:
|
|
95
|
+
return any("cache_control" in m for m in messages)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def prepare(messages: list, model: str, dial: float, has_tools: bool) -> tuple:
|
|
99
|
+
"""Apply the lever stack for the given dial. BETA.
|
|
100
|
+
|
|
101
|
+
Returns (optimized_messages, extra_payload, plan). The input list is
|
|
102
|
+
never mutated. dial 0 returns everything untouched.
|
|
103
|
+
"""
|
|
104
|
+
plan = {
|
|
105
|
+
"dial": dial,
|
|
106
|
+
"classification": "standard",
|
|
107
|
+
"levers_applied": [],
|
|
108
|
+
"tokens_pruned_est": 0,
|
|
109
|
+
"audit": [],
|
|
110
|
+
}
|
|
111
|
+
if not dial or dial <= 0:
|
|
112
|
+
return messages, {}, plan
|
|
113
|
+
|
|
114
|
+
out = copy.deepcopy(messages)
|
|
115
|
+
plan["classification"] = _classify(out, has_tools)
|
|
116
|
+
|
|
117
|
+
# lever: tool result pruning (dial 0.2+). Old tool outputs were already
|
|
118
|
+
# consumed by the model in the turn they answered; the full payload is
|
|
119
|
+
# dead weight on every later request.
|
|
120
|
+
if dial >= 0.2:
|
|
121
|
+
cutoff = len(out) - _KEEP_RECENT_MESSAGES
|
|
122
|
+
chars_removed = 0
|
|
123
|
+
for i in range(max(cutoff, 0)):
|
|
124
|
+
message = out[i]
|
|
125
|
+
content = message.get("content")
|
|
126
|
+
if (
|
|
127
|
+
message.get("role") == "tool"
|
|
128
|
+
and isinstance(content, str)
|
|
129
|
+
and len(content) > _TRUNCATE_TO_CHARS * 2
|
|
130
|
+
):
|
|
131
|
+
digest = hashlib.sha256(content.encode()).hexdigest()
|
|
132
|
+
truncated = (
|
|
133
|
+
content[:_TRUNCATE_TO_CHARS]
|
|
134
|
+
+ f"\n[mesh: pruned {len(content) - _TRUNCATE_TO_CHARS} chars "
|
|
135
|
+
"of consumed tool output]"
|
|
136
|
+
)
|
|
137
|
+
message["content"] = truncated
|
|
138
|
+
chars_removed += len(content) - len(truncated)
|
|
139
|
+
plan["audit"].append({
|
|
140
|
+
"lever": "tool_result_pruning",
|
|
141
|
+
"action": f"truncated tool result at message {i}",
|
|
142
|
+
"content_sha256": digest,
|
|
143
|
+
})
|
|
144
|
+
if chars_removed:
|
|
145
|
+
plan["tokens_pruned_est"] = -(-chars_removed // 4)
|
|
146
|
+
plan["levers_applied"].append("tool_result_pruning")
|
|
147
|
+
|
|
148
|
+
# lever: cache_control injection (dial 0+). Skips entirely when the
|
|
149
|
+
# client placed its own breakpoints. Runs after pruning so breakpoints
|
|
150
|
+
# land on the final bytes.
|
|
151
|
+
if not _has_client_breakpoints(out):
|
|
152
|
+
minimum = _cache_minimum(model)
|
|
153
|
+
applied = False
|
|
154
|
+
if out and out[0].get("role") == "system":
|
|
155
|
+
first_tokens = _est_tokens(out[0].get("content"))
|
|
156
|
+
if first_tokens >= minimum:
|
|
157
|
+
out[0]["cache_control"] = {"type": "ephemeral"}
|
|
158
|
+
plan["audit"].append({
|
|
159
|
+
"lever": "cache_injection",
|
|
160
|
+
"action": f"breakpoint on system message (~{first_tokens} tok)",
|
|
161
|
+
})
|
|
162
|
+
applied = True
|
|
163
|
+
if len(out) >= 3:
|
|
164
|
+
prefix_tokens = sum(_msg_tokens(m) for m in out[:-1])
|
|
165
|
+
anchor = out[-2]
|
|
166
|
+
if prefix_tokens >= minimum and "cache_control" not in anchor:
|
|
167
|
+
anchor["cache_control"] = {"type": "ephemeral"}
|
|
168
|
+
plan["audit"].append({
|
|
169
|
+
"lever": "cache_injection",
|
|
170
|
+
"action": f"breakpoint on history (~{prefix_tokens} tok prefix)",
|
|
171
|
+
})
|
|
172
|
+
applied = True
|
|
173
|
+
if applied:
|
|
174
|
+
plan["levers_applied"].append("cache_injection")
|
|
175
|
+
|
|
176
|
+
# lever: max_tokens default per task class (dial 0+). A backstop against
|
|
177
|
+
# runaway generation, applied only because the CLI does not set one.
|
|
178
|
+
extra = {"max_tokens": _MAX_TOKENS_DEFAULTS[plan["classification"]]}
|
|
179
|
+
plan["levers_applied"].append("max_tokens_default")
|
|
180
|
+
plan["audit"].append({
|
|
181
|
+
"lever": "max_tokens_default",
|
|
182
|
+
"action": f"max_tokens={extra['max_tokens']} for {plan['classification']} task",
|
|
183
|
+
})
|
|
184
|
+
|
|
185
|
+
return out, extra, plan
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def savings_line(plan: dict, usage: dict) -> str:
|
|
189
|
+
"""One-line honest savings summary for the post-turn status line.
|
|
190
|
+
|
|
191
|
+
Only reports what is measurable: pruned tokens (chars/4 estimate) and
|
|
192
|
+
cache fields when the gateway surfaces them in usage. No counterfactual
|
|
193
|
+
guessing.
|
|
194
|
+
"""
|
|
195
|
+
if not plan or not plan.get("levers_applied"):
|
|
196
|
+
return ""
|
|
197
|
+
parts = []
|
|
198
|
+
pruned = plan.get("tokens_pruned_est", 0)
|
|
199
|
+
if pruned:
|
|
200
|
+
parts.append(f"~{pruned} tok pruned")
|
|
201
|
+
usage = usage or {}
|
|
202
|
+
# explicit anthropic-style field first, OpenAI convention as fallback
|
|
203
|
+
cache_read = (
|
|
204
|
+
usage.get("cache_read_input_tokens")
|
|
205
|
+
or (usage.get("prompt_tokens_details") or {}).get("cached_tokens")
|
|
206
|
+
or 0
|
|
207
|
+
)
|
|
208
|
+
if cache_read:
|
|
209
|
+
parts.append(f"{cache_read} tok from cache (90% off)")
|
|
210
|
+
if "cache_injection" in plan["levers_applied"] and not cache_read:
|
|
211
|
+
parts.append("cache breakpoints set")
|
|
212
|
+
detail = ", ".join(parts) if parts else ", ".join(plan["levers_applied"])
|
|
213
|
+
return f"⚡ optimize beta (dial {plan['dial']}): {detail}"
|
meshapi/tools.py
CHANGED
|
@@ -284,8 +284,13 @@ def execute(name: str, arguments: dict) -> str:
|
|
|
284
284
|
out, _ = proc.communicate(timeout=BASH_TIMEOUT)
|
|
285
285
|
except subprocess.TimeoutExpired:
|
|
286
286
|
try:
|
|
287
|
-
os.killpg
|
|
288
|
-
|
|
287
|
+
# os.killpg + signal.SIGKILL are POSIX-only. On Windows
|
|
288
|
+
# there's no process group, so kill the child directly.
|
|
289
|
+
if hasattr(os, "killpg"):
|
|
290
|
+
os.killpg(proc.pid, signal.SIGKILL)
|
|
291
|
+
else:
|
|
292
|
+
proc.kill() # Windows: TerminateProcess on the child
|
|
293
|
+
except (ProcessLookupError, OSError):
|
|
289
294
|
pass
|
|
290
295
|
proc.communicate() # reap zombie
|
|
291
296
|
return (
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: meshapi-code
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.5
|
|
4
4
|
Summary: Terminal chat for Mesh API — OpenAI-compatible LLM gateway
|
|
5
5
|
Project-URL: Homepage, https://meshapi.ai
|
|
6
6
|
Project-URL: Documentation, https://docs.meshapi.ai
|
|
@@ -76,6 +76,44 @@ Get a key at [meshapi.ai](https://meshapi.ai).
|
|
|
76
76
|
- **Persistent input history** — up-arrow recalls past prompts across sessions.
|
|
77
77
|
- **Config + env-var override** — `~/.meshapi/config.json`, `MESHAPI_API_KEY`.
|
|
78
78
|
|
|
79
|
+
## Mesh Optimize (beta)
|
|
80
|
+
|
|
81
|
+
> **Beta feature.** Off by default. The lever stack, savings math, and command surface may change between releases. `/optimize off` bypasses everything.
|
|
82
|
+
|
|
83
|
+
One dial that cuts token spend on every request the CLI sends. Same idea as a thermostat: you pick how aggressive, the levers underneath are automatic.
|
|
84
|
+
|
|
85
|
+
```
|
|
86
|
+
/optimize 0.3 enable at dial 0.3
|
|
87
|
+
/optimize off disable (requests pass through untouched)
|
|
88
|
+
/optimize show current setting and help
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
What the dial does:
|
|
92
|
+
|
|
93
|
+
| dial | levers | quality impact |
|
|
94
|
+
|---|---|---|
|
|
95
|
+
| 0 | off, byte-identical passthrough | none |
|
|
96
|
+
| 0 to 0.2 | prompt cache breakpoint injection on stable prefixes, max_tokens defaults per task class | none |
|
|
97
|
+
| 0.2 to 0.95 | plus pruning of tool results the model already consumed in earlier turns | minimal |
|
|
98
|
+
|
|
99
|
+
Why this matters in a tool-calling REPL specifically: every turn re-sends the whole conversation, including every old `run_bash` output and file dump. A 5000-line test log from ten turns ago is billed again on every request after it. The pruning lever truncates those consumed outputs (keeping the last 4 messages untouched), and the cache lever marks the stable conversation prefix so the gateway can serve it at the provider's 90% cache discount instead of full price.
|
|
100
|
+
|
|
101
|
+
After each turn the status line reports what actually happened, honestly:
|
|
102
|
+
|
|
103
|
+
```
|
|
104
|
+
anthropic/claude-opus-4.8 • 3122→214 tok • $0.021 • session $0.084 • 6.1s
|
|
105
|
+
⚡ optimize beta (dial 0.3): ~4888 tok pruned, cache breakpoints set
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Notes:
|
|
109
|
+
|
|
110
|
+
- Works with every model Mesh serves, including `anthropic/claude-opus-4.8` and `anthropic/claude-fable-5`. Per-model rules are respected automatically (cache minimums differ per model; below the minimum no breakpoint is injected because it would do nothing).
|
|
111
|
+
- Savings are only claimed when measurable: pruned tokens are a chars/4 estimate, cache reads are reported only when the gateway surfaces them in `usage`.
|
|
112
|
+
- If the gateway rejects an optimized request for any reason, the CLI automatically retries the raw request and tells you. The beta can never be the reason a turn fails.
|
|
113
|
+
- Everything pruned is logged with a sha256 of the original content, so "why did the model forget X" has an answer.
|
|
114
|
+
- Reference implementation, tests, and design notes: [mesh-optimize on GitHub](https://github.com/raushan-aifiesta/mesh-optimize).
|
|
115
|
+
- New to Mesh? Get an API key at [app.meshapi.ai](https://app.meshapi.ai/). One key, 300+ models, and the optimizer works on all of them.
|
|
116
|
+
|
|
79
117
|
## Tool calling
|
|
80
118
|
|
|
81
119
|
When tools are enabled, the model can call:
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
meshapi/__init__.py,sha256=ErkLkI2TDBX1OIqi2GGa20CPeu4ZculEi-XffRbLU6M,22
|
|
2
|
+
meshapi/__main__.py,sha256=MSmt_5Xg84uHqzTN38JwgseJK8rsJn_11A8WD99VtEo,61
|
|
3
|
+
meshapi/attachments.py,sha256=WWepawjhA2tPm_45TX1Jura6z3q0JC3lSzCF-g_DsnA,6950
|
|
4
|
+
meshapi/cli.py,sha256=bqFjQLKWssKUAJt-W5OVSYORQ3aV7e0VRZ2p2s_qCyQ,49484
|
|
5
|
+
meshapi/client.py,sha256=5lJeRyokW08YwswbPXfBnH07b3-s0l9CBgEkOKCqBjY,4436
|
|
6
|
+
meshapi/commands.py,sha256=ZYoW7SmULOtB4q-DZXRz9pB-UdW0aTjFqxSeRlMnkb0,8598
|
|
7
|
+
meshapi/config.py,sha256=YchDAS75gH4O2uaVf1JzlrQFSfHCsFPB1sahjYxGf5k,4076
|
|
8
|
+
meshapi/keywatcher.py,sha256=tWVSLWZY-p08CcOd10Xvf5TrMGfjDaKDzYJRSfe4kPo,8057
|
|
9
|
+
meshapi/optimize.py,sha256=HsvYCzvTAm62aBW3KB1N9B2cC-XjLO86huhmoUR5bCs,7817
|
|
10
|
+
meshapi/permissions.py,sha256=xyRyob-M_zYGak1rn5T1xqv3iHcY-n6z35QnFwWm3zI,2451
|
|
11
|
+
meshapi/plan.py,sha256=A7hYfGF1tajTNwMOZ1A0V79DspYwM7jrecbe3OsUTxM,3460
|
|
12
|
+
meshapi/render.py,sha256=VwgDbYSElwEJ0WhSMpRZ8Tw_EA0A09s8D4yVh_nUL3o,4737
|
|
13
|
+
meshapi/safety.py,sha256=OS9_FDAz-DcNMo6zjoz4VQSXAGczJFCZGyWYrEexifk,10795
|
|
14
|
+
meshapi/statusbar.py,sha256=PnTLrgvcFna5_1uA5whdsdvwyhHTDpfRcuq4UoURmZk,4144
|
|
15
|
+
meshapi/tools.py,sha256=fFeZHtorLcS4ckyl2YOAc7r0YKbrT6VmRxjE27KQQX0,15455
|
|
16
|
+
meshapi_code-0.4.5.dist-info/METADATA,sha256=yr0Ws6eWjVYhiacJnWiU7ckZ0Rw5LRkFHjOz8lUs10k,10117
|
|
17
|
+
meshapi_code-0.4.5.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
18
|
+
meshapi_code-0.4.5.dist-info/entry_points.txt,sha256=ZCXZ_SgrhWIQEHSjAXz0pUlyGbIQKZ68vp_Cg1Y0rME,45
|
|
19
|
+
meshapi_code-0.4.5.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
20
|
+
meshapi_code-0.4.5.dist-info/licenses/NOTICE,sha256=wF-6Apse4eVIOpbNP3WLtTaOJClNFK7Jok2BnUvSo9U,191
|
|
21
|
+
meshapi_code-0.4.5.dist-info/RECORD,,
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
meshapi/__init__.py,sha256=6G_giX6Ucuweo7w5OiftoXmbNLoqiU_soXJoU8aiLmY,22
|
|
2
|
-
meshapi/__main__.py,sha256=MSmt_5Xg84uHqzTN38JwgseJK8rsJn_11A8WD99VtEo,61
|
|
3
|
-
meshapi/attachments.py,sha256=WWepawjhA2tPm_45TX1Jura6z3q0JC3lSzCF-g_DsnA,6950
|
|
4
|
-
meshapi/cli.py,sha256=D571-Jd9eHFnOHA8-mZPuAAzGxXmI_wErWAJbaiGB1E,48334
|
|
5
|
-
meshapi/client.py,sha256=Rtc-8W9XncxPlV6qQ9I_c25BizyBHYNiIy8Eb3kSaEw,2920
|
|
6
|
-
meshapi/commands.py,sha256=LifH9RCdHmR7Av_30mggpmZgdS5V9v529gyiDjk4Lls,6767
|
|
7
|
-
meshapi/config.py,sha256=K478RB4YFcXePmcJO4xIg8jwUW1TgK1hz0Znut3lV_o,3909
|
|
8
|
-
meshapi/keywatcher.py,sha256=tWVSLWZY-p08CcOd10Xvf5TrMGfjDaKDzYJRSfe4kPo,8057
|
|
9
|
-
meshapi/permissions.py,sha256=xyRyob-M_zYGak1rn5T1xqv3iHcY-n6z35QnFwWm3zI,2451
|
|
10
|
-
meshapi/plan.py,sha256=A7hYfGF1tajTNwMOZ1A0V79DspYwM7jrecbe3OsUTxM,3460
|
|
11
|
-
meshapi/render.py,sha256=VwgDbYSElwEJ0WhSMpRZ8Tw_EA0A09s8D4yVh_nUL3o,4737
|
|
12
|
-
meshapi/safety.py,sha256=OS9_FDAz-DcNMo6zjoz4VQSXAGczJFCZGyWYrEexifk,10795
|
|
13
|
-
meshapi/statusbar.py,sha256=PnTLrgvcFna5_1uA5whdsdvwyhHTDpfRcuq4UoURmZk,4144
|
|
14
|
-
meshapi/tools.py,sha256=dMu3jr7HoKIRvHSSF_NL75QlZ8vmkwYR_3iKuwxv1Ec,15138
|
|
15
|
-
meshapi_code-0.4.4.dist-info/METADATA,sha256=YYrxnHlkt8CygfDff4eB74TxRiLxLqWAGZa2_LJpTj4,7595
|
|
16
|
-
meshapi_code-0.4.4.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
17
|
-
meshapi_code-0.4.4.dist-info/entry_points.txt,sha256=ZCXZ_SgrhWIQEHSjAXz0pUlyGbIQKZ68vp_Cg1Y0rME,45
|
|
18
|
-
meshapi_code-0.4.4.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
19
|
-
meshapi_code-0.4.4.dist-info/licenses/NOTICE,sha256=wF-6Apse4eVIOpbNP3WLtTaOJClNFK7Jok2BnUvSo9U,191
|
|
20
|
-
meshapi_code-0.4.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|