secondwind 0.1.0__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- secondwind/__init__.py +204 -0
- secondwind/_lib/secondwind.dll +0 -0
- secondwind/agno.py +52 -0
- secondwind/asgi.py +81 -0
- secondwind/cursor.py +70 -0
- secondwind/langchain.py +83 -0
- secondwind/litellm.py +38 -0
- secondwind/strands.py +48 -0
- secondwind-0.1.0.dist-info/METADATA +68 -0
- secondwind-0.1.0.dist-info/RECORD +12 -0
- secondwind-0.1.0.dist-info/WHEEL +5 -0
- secondwind-0.1.0.dist-info/top_level.txt +1 -0
secondwind/__init__.py
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""secondwind: lossless, provable, composable LLM context compression.
|
|
2
|
+
|
|
3
|
+
Thin ctypes binding over the bundled C ABI (no native extension); the first caller of a C ABI
|
|
4
|
+
every language calls the same way.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import ctypes
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _library_path() -> str:
|
|
13
|
+
if os.environ.get("SECONDWIND_LIB"):
|
|
14
|
+
return os.environ["SECONDWIND_LIB"]
|
|
15
|
+
here = os.path.dirname(__file__)
|
|
16
|
+
names = ("libsecondwind.dylib", "libsecondwind.so", "secondwind.dll")
|
|
17
|
+
# Bundled inside the wheel.
|
|
18
|
+
for name in names:
|
|
19
|
+
bundled = os.path.join(here, "_lib", name)
|
|
20
|
+
if os.path.exists(bundled):
|
|
21
|
+
return bundled
|
|
22
|
+
# Dev fallback: source-checkout cargo target dir.
|
|
23
|
+
target = os.path.join(here, "..", "..", "..", "target", "release")
|
|
24
|
+
for name in names:
|
|
25
|
+
dev = os.path.join(target, name)
|
|
26
|
+
if os.path.exists(dev):
|
|
27
|
+
return dev
|
|
28
|
+
raise OSError("secondwind native library not found; build it or set SECONDWIND_LIB")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
_lib = ctypes.CDLL(_library_path())
|
|
32
|
+
_lib.sw_abi_version.restype = ctypes.c_uint32
|
|
33
|
+
for _fn in ("sw_compress", "sw_verify"):
|
|
34
|
+
f = getattr(_lib, _fn)
|
|
35
|
+
f.argtypes = [ctypes.c_char_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)]
|
|
36
|
+
f.restype = ctypes.POINTER(ctypes.c_ubyte)
|
|
37
|
+
_lib.sw_free.argtypes = [ctypes.POINTER(ctypes.c_ubyte), ctypes.c_size_t]
|
|
38
|
+
_lib.sw_session_new.argtypes = [ctypes.c_char_p, ctypes.c_size_t]
|
|
39
|
+
_lib.sw_session_new.restype = ctypes.c_void_p
|
|
40
|
+
_lib.sw_session_free.argtypes = [ctypes.c_void_p]
|
|
41
|
+
_lib.sw_rewrite.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)]
|
|
42
|
+
_lib.sw_rewrite.restype = ctypes.POINTER(ctypes.c_ubyte)
|
|
43
|
+
|
|
44
|
+
# Host offload backend: put(ctx, id, id_len, val, val_len) -> int; get(ctx, id, id_len, out_len) -> bytes*.
|
|
45
|
+
_PUT = ctypes.CFUNCTYPE(
|
|
46
|
+
ctypes.c_int, ctypes.c_void_p, ctypes.POINTER(ctypes.c_ubyte), ctypes.c_size_t,
|
|
47
|
+
ctypes.POINTER(ctypes.c_ubyte), ctypes.c_size_t,
|
|
48
|
+
)
|
|
49
|
+
_GET = ctypes.CFUNCTYPE(
|
|
50
|
+
ctypes.c_void_p, ctypes.c_void_p, ctypes.POINTER(ctypes.c_ubyte), ctypes.c_size_t,
|
|
51
|
+
ctypes.POINTER(ctypes.c_size_t),
|
|
52
|
+
)
|
|
53
|
+
_lib.sw_session_new_with_store.argtypes = [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_void_p, _PUT, _GET]
|
|
54
|
+
_lib.sw_session_new_with_store.restype = ctypes.c_void_p
|
|
55
|
+
|
|
56
|
+
# Host codec: encode/decode take input bytes, return an output ptr (len via out_len) or null; same
|
|
57
|
+
# shape as the store's get callback.
|
|
58
|
+
_CODEC = ctypes.CFUNCTYPE(
|
|
59
|
+
ctypes.c_void_p, ctypes.c_void_p, ctypes.POINTER(ctypes.c_ubyte), ctypes.c_size_t,
|
|
60
|
+
ctypes.POINTER(ctypes.c_size_t),
|
|
61
|
+
)
|
|
62
|
+
_lib.sw_session_new_with_codec.argtypes = [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_void_p, _CODEC, _CODEC]
|
|
63
|
+
_lib.sw_session_new_with_codec.restype = ctypes.c_void_p
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _call(fn, payload: dict) -> dict:
|
|
67
|
+
data = json.dumps(payload).encode("utf-8")
|
|
68
|
+
out_len = ctypes.c_size_t(0)
|
|
69
|
+
ptr = fn(data, len(data), ctypes.byref(out_len))
|
|
70
|
+
try:
|
|
71
|
+
return json.loads(ctypes.string_at(ptr, out_len.value))
|
|
72
|
+
finally:
|
|
73
|
+
_lib.sw_free(ptr, out_len.value)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def abi_version() -> int:
|
|
77
|
+
return _lib.sw_abi_version()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def compress(block: str, model: str | None = None) -> dict:
|
|
81
|
+
"""Losslessly compress one tool-output block. Returns the outcome dict."""
|
|
82
|
+
payload = {"block": block}
|
|
83
|
+
if model:
|
|
84
|
+
payload["model"] = model
|
|
85
|
+
return _call(_lib.sw_compress, payload)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def verify(wire: str, cert_hash: str) -> bool:
|
|
89
|
+
"""Independently confirm a wire is lossless against its certificate hash (no trust required)."""
|
|
90
|
+
return _call(_lib.sw_verify, {"wire": wire, "hash": cert_hash}).get("ok", False)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class Session:
|
|
94
|
+
"""Per-conversation handle: freeze memory keeps resends byte-identical so only aged blocks
|
|
95
|
+
offload. `home` books ledger events for `secondwind proof --home <dir>`."""
|
|
96
|
+
|
|
97
|
+
def __init__(self, resolver=None, offload_dir=None, home=None, store=None, proposers=True, codec=None):
|
|
98
|
+
"""`store`: any object with put(id, value) -> bool and get(id) -> str | None, so offload can
|
|
99
|
+
be backed by Redis/S3/a database. `proposers` toggles the best-of-N codec search (on; off is a
|
|
100
|
+
cost/latency choice, output is proven lossless either way). `codec`: any object with
|
|
101
|
+
encode(text) and decode(wire); it competes in the search and is dropped unless
|
|
102
|
+
decode(encode(x)) == x per block."""
|
|
103
|
+
config: dict = {"proposers": bool(proposers)}
|
|
104
|
+
if resolver:
|
|
105
|
+
config["resolver"] = resolver
|
|
106
|
+
if offload_dir:
|
|
107
|
+
config["offload_dir"] = offload_dir
|
|
108
|
+
if home:
|
|
109
|
+
config["home"] = home
|
|
110
|
+
data = json.dumps(config).encode("utf-8")
|
|
111
|
+
if codec is not None:
|
|
112
|
+
self._ptr = self._open_with_codec(data, codec)
|
|
113
|
+
elif store is None:
|
|
114
|
+
self._ptr = _lib.sw_session_new(data, len(data))
|
|
115
|
+
else:
|
|
116
|
+
self._ptr = self._open_with_store(data, store)
|
|
117
|
+
self._totals = {"requests": 0, "blocks_rewritten": 0, "blocks_first_seen": 0, "tokens_saved": 0}
|
|
118
|
+
|
|
119
|
+
def _open_with_codec(self, config: bytes, codec):
|
|
120
|
+
self._codec = codec
|
|
121
|
+
self._enc_buf = None # keep each result alive until secondwind copies it
|
|
122
|
+
self._dec_buf = None
|
|
123
|
+
|
|
124
|
+
def side(method, hold):
|
|
125
|
+
def cb(_ctx, in_ptr, in_len, out_len):
|
|
126
|
+
try:
|
|
127
|
+
result = method(ctypes.string_at(in_ptr, in_len).decode("utf-8"))
|
|
128
|
+
except Exception:
|
|
129
|
+
result = None
|
|
130
|
+
if not isinstance(result, str):
|
|
131
|
+
out_len[0] = 0
|
|
132
|
+
return None
|
|
133
|
+
raw = result.encode("utf-8")
|
|
134
|
+
buf = (ctypes.c_ubyte * len(raw)).from_buffer_copy(raw)
|
|
135
|
+
setattr(self, hold, buf)
|
|
136
|
+
out_len[0] = len(raw)
|
|
137
|
+
return ctypes.addressof(buf)
|
|
138
|
+
|
|
139
|
+
return cb
|
|
140
|
+
|
|
141
|
+
self._enc_cb = _CODEC(side(codec.encode, "_enc_buf"))
|
|
142
|
+
self._dec_cb = _CODEC(side(codec.decode, "_dec_buf"))
|
|
143
|
+
return _lib.sw_session_new_with_codec(config, len(config), None, self._enc_cb, self._dec_cb)
|
|
144
|
+
|
|
145
|
+
def _open_with_store(self, config: bytes, store):
|
|
146
|
+
self._store = store
|
|
147
|
+
self._get_buf = None # keeps the last get result alive until the next call
|
|
148
|
+
|
|
149
|
+
def put_cb(_ctx, id_ptr, id_len, val_ptr, val_len):
|
|
150
|
+
try:
|
|
151
|
+
key = ctypes.string_at(id_ptr, id_len).decode("utf-8")
|
|
152
|
+
val = ctypes.string_at(val_ptr, val_len).decode("utf-8")
|
|
153
|
+
return 1 if store.put(key, val) else 0
|
|
154
|
+
except Exception:
|
|
155
|
+
return 0
|
|
156
|
+
|
|
157
|
+
def get_cb(_ctx, id_ptr, id_len, out_len):
|
|
158
|
+
try:
|
|
159
|
+
key = ctypes.string_at(id_ptr, id_len).decode("utf-8")
|
|
160
|
+
val = store.get(key)
|
|
161
|
+
except Exception:
|
|
162
|
+
val = None
|
|
163
|
+
if val is None:
|
|
164
|
+
out_len[0] = 0
|
|
165
|
+
return None
|
|
166
|
+
raw = val.encode("utf-8")
|
|
167
|
+
self._get_buf = (ctypes.c_ubyte * len(raw)).from_buffer_copy(raw)
|
|
168
|
+
out_len[0] = len(raw)
|
|
169
|
+
return ctypes.addressof(self._get_buf)
|
|
170
|
+
|
|
171
|
+
# Keep the callback objects alive on self, or they are garbage-collected and the FFI crashes.
|
|
172
|
+
self._put_cb = _PUT(put_cb)
|
|
173
|
+
self._get_cb = _GET(get_cb)
|
|
174
|
+
return _lib.sw_session_new_with_store(config, len(config), None, self._put_cb, self._get_cb)
|
|
175
|
+
|
|
176
|
+
def rewrite(self, request: dict) -> dict:
|
|
177
|
+
"""Compress every tool-output block in a whole LLM request. Returns {"request", "stats"}."""
|
|
178
|
+
data = json.dumps(request).encode("utf-8")
|
|
179
|
+
out_len = ctypes.c_size_t(0)
|
|
180
|
+
ptr = _lib.sw_rewrite(self._ptr, data, len(data), ctypes.byref(out_len))
|
|
181
|
+
try:
|
|
182
|
+
out = json.loads(ctypes.string_at(ptr, out_len.value))
|
|
183
|
+
finally:
|
|
184
|
+
_lib.sw_free(ptr, out_len.value)
|
|
185
|
+
stats = out.get("stats", {})
|
|
186
|
+
self._totals["requests"] += 1
|
|
187
|
+
for key in ("blocks_rewritten", "blocks_first_seen", "tokens_saved"):
|
|
188
|
+
self._totals[key] += stats.get(key, 0)
|
|
189
|
+
return out
|
|
190
|
+
|
|
191
|
+
def stats(self) -> dict:
|
|
192
|
+
"""Running totals for this session (no proxy needed)."""
|
|
193
|
+
return dict(self._totals)
|
|
194
|
+
|
|
195
|
+
def close(self):
|
|
196
|
+
if self._ptr:
|
|
197
|
+
_lib.sw_session_free(self._ptr)
|
|
198
|
+
self._ptr = None
|
|
199
|
+
|
|
200
|
+
def __enter__(self):
|
|
201
|
+
return self
|
|
202
|
+
|
|
203
|
+
def __exit__(self, *_):
|
|
204
|
+
self.close()
|
|
Binary file
|
secondwind/agno.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Agno integration: lossless replacement for Agno's tool-result compression.
|
|
2
|
+
|
|
3
|
+
from agno.agent import Agent
|
|
4
|
+
from secondwind.agno import SecondwindCompressionManager
|
|
5
|
+
|
|
6
|
+
agent = Agent(
|
|
7
|
+
model=...,
|
|
8
|
+
compress_tool_results=True,
|
|
9
|
+
compression_manager=SecondwindCompressionManager(compress_tool_results=True),
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
Keeps Agno's gating (when/which results to compress) but swaps its lossy per-result LLM summarizer
|
|
13
|
+
for secondwind: lossless, no model call. Any failure returns None so Agno keeps the original.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
from agno.compression import CompressionManager as _CompressionManager
|
|
18
|
+
|
|
19
|
+
_AGNO = True
|
|
20
|
+
except ImportError: # agno optional
|
|
21
|
+
_CompressionManager = object
|
|
22
|
+
_AGNO = False
|
|
23
|
+
|
|
24
|
+
from . import Session
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class SecondwindCompressionManager(_CompressionManager):
|
|
28
|
+
_sw_session = None
|
|
29
|
+
|
|
30
|
+
def _session(self):
|
|
31
|
+
if self._sw_session is None:
|
|
32
|
+
self._sw_session = Session()
|
|
33
|
+
return self._sw_session
|
|
34
|
+
|
|
35
|
+
def _compress_lossless(self, content):
|
|
36
|
+
if not isinstance(content, str) or not content:
|
|
37
|
+
return None
|
|
38
|
+
try:
|
|
39
|
+
out = self._session().rewrite(
|
|
40
|
+
{"model": "", "messages": [{"role": "tool", "tool_call_id": "t", "content": content}]}
|
|
41
|
+
)
|
|
42
|
+
new = out["request"]["messages"][0]["content"]
|
|
43
|
+
return new if isinstance(new, str) and new != content else None
|
|
44
|
+
except Exception: # fail open: Agno keeps the original
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
# Override both engine hooks so Agno's compress()/acompress() call secondwind, not an LLM.
|
|
48
|
+
def _compress_tool_result(self, tool_msg, *args, **kwargs):
|
|
49
|
+
return self._compress_lossless(getattr(tool_msg, "content", None))
|
|
50
|
+
|
|
51
|
+
async def _acompress_tool_result(self, tool_msg, *args, **kwargs):
|
|
52
|
+
return self._compress_lossless(getattr(tool_msg, "content", None))
|
secondwind/asgi.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""ASGI middleware: compress LLM tool outputs for a self-hosted gateway with one line.
|
|
2
|
+
|
|
3
|
+
from secondwind.asgi import SecondwindMiddleware
|
|
4
|
+
app.add_middleware(SecondwindMiddleware) # Starlette / FastAPI
|
|
5
|
+
# or wrap any ASGI app directly: app = SecondwindMiddleware(app)
|
|
6
|
+
|
|
7
|
+
Rewrites tool outputs in a JSON chat-completion body in-process before your app forwards upstream.
|
|
8
|
+
Only touches POST requests whose JSON body has `messages`; any failure falls back to the original body.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
|
|
13
|
+
from . import Session
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
async def _read_body(receive):
|
|
17
|
+
chunks = []
|
|
18
|
+
while True:
|
|
19
|
+
message = await receive()
|
|
20
|
+
if message["type"] == "http.request":
|
|
21
|
+
chunks.append(message.get("body", b""))
|
|
22
|
+
if not message.get("more_body", False):
|
|
23
|
+
break
|
|
24
|
+
elif message["type"] == "http.disconnect":
|
|
25
|
+
break
|
|
26
|
+
return b"".join(chunks)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _replay(body):
|
|
30
|
+
# Replay the (possibly rewritten) body as one request event, then disconnect.
|
|
31
|
+
done = False
|
|
32
|
+
|
|
33
|
+
async def receive():
|
|
34
|
+
nonlocal done
|
|
35
|
+
if not done:
|
|
36
|
+
done = True
|
|
37
|
+
return {"type": "http.request", "body": body, "more_body": False}
|
|
38
|
+
return {"type": "http.disconnect"}
|
|
39
|
+
|
|
40
|
+
return receive
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class SecondwindMiddleware:
|
|
44
|
+
def __init__(self, app, resolver=None, home=None, paths=None):
|
|
45
|
+
"""`paths`: a prefix (or tuple) to limit rewriting to your LLM route; default touches any
|
|
46
|
+
POST whose JSON body has `messages`."""
|
|
47
|
+
self.app = app
|
|
48
|
+
self._session = Session(resolver=resolver, home=home)
|
|
49
|
+
self._paths = (paths,) if isinstance(paths, str) else tuple(paths) if paths else None
|
|
50
|
+
|
|
51
|
+
async def __call__(self, scope, receive, send):
|
|
52
|
+
if scope["type"] != "http" or scope.get("method") != "POST":
|
|
53
|
+
return await self.app(scope, receive, send)
|
|
54
|
+
if self._paths and not scope["path"].startswith(self._paths):
|
|
55
|
+
return await self.app(scope, receive, send)
|
|
56
|
+
|
|
57
|
+
body = await _read_body(receive)
|
|
58
|
+
new_body = self._maybe_rewrite(body)
|
|
59
|
+
if new_body is body:
|
|
60
|
+
return await self.app(scope, _replay(body), send)
|
|
61
|
+
|
|
62
|
+
# Body length changed: correct Content-Length before the app reads it.
|
|
63
|
+
headers = [(k, v) for (k, v) in scope["headers"] if k.lower() != b"content-length"]
|
|
64
|
+
headers.append((b"content-length", str(len(new_body)).encode()))
|
|
65
|
+
return await self.app({**scope, "headers": headers}, _replay(new_body), send)
|
|
66
|
+
|
|
67
|
+
def _maybe_rewrite(self, body):
|
|
68
|
+
try:
|
|
69
|
+
payload = json.loads(body)
|
|
70
|
+
except Exception:
|
|
71
|
+
return body
|
|
72
|
+
if not isinstance(payload, dict) or "messages" not in payload:
|
|
73
|
+
return body
|
|
74
|
+
try:
|
|
75
|
+
out = self._session.rewrite(payload)
|
|
76
|
+
return json.dumps(out["request"]).encode("utf-8")
|
|
77
|
+
except Exception: # never break the request
|
|
78
|
+
return body
|
|
79
|
+
|
|
80
|
+
def close(self):
|
|
81
|
+
self._session.close()
|
secondwind/cursor.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Cursor integration: a postToolUse hook that losslessly compresses MCP tool output.
|
|
2
|
+
|
|
3
|
+
Cursor's one supported output-rewrite seam is `postToolUse` -> `updated_mcp_tool_output`. This module
|
|
4
|
+
IS that hook: reads Cursor's JSON on stdin, compresses the tool output, writes the replacement on
|
|
5
|
+
stdout. Any error or non-compressible output returns {} so the model sees the original.
|
|
6
|
+
|
|
7
|
+
Configure ~/.cursor/hooks.json (or a project .cursor/hooks.json):
|
|
8
|
+
|
|
9
|
+
{
|
|
10
|
+
"version": 1,
|
|
11
|
+
"hooks": { "postToolUse": [ { "command": "python -m secondwind.cursor" } ] }
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
For chat / plan mode (a different seam), point Cursor's "Override OpenAI Base URL" at a running
|
|
15
|
+
`secondwind serve` proxy. Cursor's hook field names are beta and version-dependent; if yours differ,
|
|
16
|
+
set FIELD_IN / FIELD_OUT below.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import sys
|
|
21
|
+
|
|
22
|
+
from . import compress
|
|
23
|
+
|
|
24
|
+
FIELD_IN = "tool_output"
|
|
25
|
+
FIELD_OUT = "updated_mcp_tool_output"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _compress_text(text):
|
|
29
|
+
if not isinstance(text, str) or not text:
|
|
30
|
+
return None
|
|
31
|
+
try:
|
|
32
|
+
out = compress(text)
|
|
33
|
+
if out.get("kind") == "compressed":
|
|
34
|
+
return out["wire"]
|
|
35
|
+
except Exception:
|
|
36
|
+
pass
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def rewrite_output(payload):
|
|
41
|
+
"""The hook's stdout for a postToolUse payload, or {} to leave the tool output unchanged."""
|
|
42
|
+
output = payload.get(FIELD_IN)
|
|
43
|
+
if isinstance(output, str):
|
|
44
|
+
wire = _compress_text(output)
|
|
45
|
+
return {FIELD_OUT: wire} if wire else {}
|
|
46
|
+
# MCP content-block shape: {"content": [{"text": ...}, ...]}
|
|
47
|
+
if isinstance(output, dict) and isinstance(output.get("content"), list):
|
|
48
|
+
content = [dict(block) if isinstance(block, dict) else block for block in output["content"]]
|
|
49
|
+
changed = False
|
|
50
|
+
for block in content:
|
|
51
|
+
if isinstance(block, dict) and isinstance(block.get("text"), str):
|
|
52
|
+
wire = _compress_text(block["text"])
|
|
53
|
+
if wire:
|
|
54
|
+
block["text"] = wire
|
|
55
|
+
changed = True
|
|
56
|
+
return {FIELD_OUT: {**output, "content": content}} if changed else {}
|
|
57
|
+
return {}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def main():
|
|
61
|
+
try:
|
|
62
|
+
payload = json.load(sys.stdin)
|
|
63
|
+
except Exception:
|
|
64
|
+
sys.stdout.write("{}")
|
|
65
|
+
return
|
|
66
|
+
json.dump(rewrite_output(payload), sys.stdout)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
if __name__ == "__main__":
|
|
70
|
+
main()
|
secondwind/langchain.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""LangChain and LangGraph integration: compress tool-message outputs losslessly, two ways.
|
|
2
|
+
|
|
3
|
+
LCEL chain:
|
|
4
|
+
|
|
5
|
+
from secondwind.langchain import compress_tool_outputs
|
|
6
|
+
chain = compress_tool_outputs() | model # model = any LangChain chat model
|
|
7
|
+
|
|
8
|
+
LangGraph agent loop:
|
|
9
|
+
|
|
10
|
+
from langgraph.prebuilt import create_react_agent
|
|
11
|
+
from secondwind.langchain import compress_pre_model_hook
|
|
12
|
+
agent = create_react_agent(model, tools, pre_model_hook=compress_pre_model_hook())
|
|
13
|
+
|
|
14
|
+
Both convert to the OpenAI shape, rewrite tool outputs through a session (resends stay byte-identical,
|
|
15
|
+
only aged blocks offload), and map compressed content back by tool_call_id. Any failure returns the
|
|
16
|
+
messages unchanged, so neither seam can break a chain or agent.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from . import Session
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _rewrite_tool_messages(sess, model, messages):
|
|
23
|
+
"""New message list with only ToolMessage string content compressed; others are the original
|
|
24
|
+
object. On failure returns the input list itself, so list identity signals whether anything changed."""
|
|
25
|
+
from langchain_core.messages import ToolMessage, convert_to_openai_messages
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
openai_messages = convert_to_openai_messages(messages)
|
|
29
|
+
out = sess.rewrite({"model": model, "messages": openai_messages})
|
|
30
|
+
rewritten = out.get("request", {}).get("messages", [])
|
|
31
|
+
except Exception: # never break the caller
|
|
32
|
+
return messages
|
|
33
|
+
|
|
34
|
+
by_id = {
|
|
35
|
+
m.get("tool_call_id"): m.get("content")
|
|
36
|
+
for m in rewritten
|
|
37
|
+
if isinstance(m, dict) and m.get("role") == "tool"
|
|
38
|
+
}
|
|
39
|
+
result = []
|
|
40
|
+
for message in messages:
|
|
41
|
+
new_content = by_id.get(getattr(message, "tool_call_id", None))
|
|
42
|
+
if (
|
|
43
|
+
isinstance(message, ToolMessage)
|
|
44
|
+
and isinstance(message.content, str)
|
|
45
|
+
and isinstance(new_content, str)
|
|
46
|
+
and new_content != message.content
|
|
47
|
+
):
|
|
48
|
+
result.append(message.model_copy(update={"content": new_content}))
|
|
49
|
+
else:
|
|
50
|
+
result.append(message)
|
|
51
|
+
return result
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def compress_tool_outputs(session=None, model="gpt-4o", resolver=None):
|
|
55
|
+
"""LCEL runnable that compresses the tool messages passing through it, so the downstream model
|
|
56
|
+
in `compress_tool_outputs() | model` reads the compressed form."""
|
|
57
|
+
from langchain_core.runnables import RunnableLambda
|
|
58
|
+
|
|
59
|
+
sess = session or Session(resolver=resolver)
|
|
60
|
+
|
|
61
|
+
def _transform(value):
|
|
62
|
+
messages = value.to_messages() if hasattr(value, "to_messages") else list(value)
|
|
63
|
+
return _rewrite_tool_messages(sess, model, messages)
|
|
64
|
+
|
|
65
|
+
return RunnableLambda(_transform)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def compress_pre_model_hook(session=None, model="gpt-4o", resolver=None):
|
|
69
|
+
"""LangGraph pre_model_hook: compress accumulated tool outputs before each model call.
|
|
70
|
+
|
|
71
|
+
Tool results pile up in state and are re-sent every step. Returns them as `llm_input_messages` so
|
|
72
|
+
the model reads the compressed form while durable `messages` history keeps the originals
|
|
73
|
+
byte-for-byte. Returns no update when nothing compresses (or the rewrite fails)."""
|
|
74
|
+
sess = session or Session(resolver=resolver)
|
|
75
|
+
|
|
76
|
+
def _hook(state):
|
|
77
|
+
messages = state["messages"] if isinstance(state, dict) else getattr(state, "messages", [])
|
|
78
|
+
rewritten = _rewrite_tool_messages(sess, model, messages)
|
|
79
|
+
if rewritten is messages or all(a is b for a, b in zip(rewritten, messages)):
|
|
80
|
+
return {}
|
|
81
|
+
return {"llm_input_messages": rewritten}
|
|
82
|
+
|
|
83
|
+
return _hook
|
secondwind/litellm.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""LiteLLM callback: add secondwind lossless compression with one line.
|
|
2
|
+
|
|
3
|
+
import litellm
|
|
4
|
+
from secondwind.litellm import SecondwindCallback
|
|
5
|
+
litellm.callbacks = [SecondwindCallback()]
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
from litellm.integrations.custom_logger import CustomLogger as _CustomLogger
|
|
12
|
+
except ImportError: # litellm optional
|
|
13
|
+
_CustomLogger = object
|
|
14
|
+
|
|
15
|
+
from . import Session
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger("secondwind")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class SecondwindCallback(_CustomLogger):
|
|
21
|
+
"""One-line LiteLLM integration; compresses every request's tool outputs in-process, losslessly.
|
|
22
|
+
Any failure falls back to the original request."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, resolver: str | None = None, home: str | None = None):
|
|
25
|
+
super().__init__()
|
|
26
|
+
self._session = Session(resolver=resolver, home=home)
|
|
27
|
+
|
|
28
|
+
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
|
29
|
+
try:
|
|
30
|
+
out = self._session.rewrite(data)
|
|
31
|
+
if isinstance(out, dict) and "request" in out:
|
|
32
|
+
data.update(out["request"])
|
|
33
|
+
except Exception as exc: # never break the call
|
|
34
|
+
logger.warning("secondwind compression skipped: %s", exc)
|
|
35
|
+
return data
|
|
36
|
+
|
|
37
|
+
def close(self):
|
|
38
|
+
self._session.close()
|
secondwind/strands.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Strands Agents integration: compress tool results losslessly as they are produced.
|
|
2
|
+
|
|
3
|
+
from strands import Agent
|
|
4
|
+
from secondwind.strands import SecondwindHooks
|
|
5
|
+
|
|
6
|
+
agent = Agent(model=..., tools=[...], hooks=[SecondwindHooks()])
|
|
7
|
+
|
|
8
|
+
Registers on AfterToolCallEvent and rewrites each tool result's text before it reaches the model,
|
|
9
|
+
in-process. Any failure leaves the result untouched.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
try:
|
|
13
|
+
from strands.hooks import AfterToolCallEvent, HookProvider
|
|
14
|
+
|
|
15
|
+
_STRANDS = True
|
|
16
|
+
except ImportError: # strands optional
|
|
17
|
+
HookProvider = object
|
|
18
|
+
_STRANDS = False
|
|
19
|
+
|
|
20
|
+
from . import Session
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class SecondwindHooks(HookProvider):
|
|
24
|
+
def __init__(self, resolver=None):
|
|
25
|
+
self._session = Session(resolver=resolver)
|
|
26
|
+
|
|
27
|
+
def register_hooks(self, registry, **kwargs):
|
|
28
|
+
registry.add_callback(AfterToolCallEvent, self._on_after_tool_call)
|
|
29
|
+
|
|
30
|
+
def _on_after_tool_call(self, event):
|
|
31
|
+
try:
|
|
32
|
+
result = getattr(event, "result", None)
|
|
33
|
+
content = result.get("content") if isinstance(result, dict) else None
|
|
34
|
+
if not content:
|
|
35
|
+
return
|
|
36
|
+
tool_use_id = result.get("toolUseId", "t")
|
|
37
|
+
for block in content:
|
|
38
|
+
text = block.get("text") if isinstance(block, dict) else None
|
|
39
|
+
if not isinstance(text, str) or not text:
|
|
40
|
+
continue
|
|
41
|
+
out = self._session.rewrite(
|
|
42
|
+
{"model": "", "messages": [{"role": "tool", "tool_call_id": tool_use_id, "content": text}]}
|
|
43
|
+
)
|
|
44
|
+
new = out["request"]["messages"][0]["content"]
|
|
45
|
+
if isinstance(new, str) and new != text:
|
|
46
|
+
block["text"] = new # mutate the result dict in place, before it reaches the model
|
|
47
|
+
except Exception: # never break the run
|
|
48
|
+
pass
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: secondwind
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Lossless, provable, composable LLM context compression
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Provides-Extra: litellm
|
|
9
|
+
Requires-Dist: litellm; extra == "litellm"
|
|
10
|
+
|
|
11
|
+
# secondwind
|
|
12
|
+
|
|
13
|
+
Lossless, provable, composable LLM tool-output compression, in-process.
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
import secondwind
|
|
17
|
+
|
|
18
|
+
session = secondwind.Session()
|
|
19
|
+
out = session.rewrite(request) # compress a whole request's tool outputs
|
|
20
|
+
print(out["stats"]) # tokens saved
|
|
21
|
+
secondwind.compress(block) # or compress a single tool-output block
|
|
22
|
+
|
|
23
|
+
# one-line LiteLLM integration:
|
|
24
|
+
import litellm
|
|
25
|
+
from secondwind.litellm import SecondwindCallback
|
|
26
|
+
litellm.callbacks = [SecondwindCallback()]
|
|
27
|
+
|
|
28
|
+
# ASGI gateway (Starlette / FastAPI): rewrite request bodies before you forward them upstream
|
|
29
|
+
from secondwind.asgi import SecondwindMiddleware
|
|
30
|
+
app.add_middleware(SecondwindMiddleware)
|
|
31
|
+
|
|
32
|
+
# LangChain (LCEL): compress tool outputs before the model call
|
|
33
|
+
from secondwind.langchain import compress_tool_outputs
|
|
34
|
+
chain = compress_tool_outputs() | model
|
|
35
|
+
|
|
36
|
+
# LangGraph: compress the tool outputs an agent loop re-sends before each model call
|
|
37
|
+
from secondwind.langchain import compress_pre_model_hook
|
|
38
|
+
agent = create_react_agent(model, tools, pre_model_hook=compress_pre_model_hook())
|
|
39
|
+
|
|
40
|
+
# Agno: a lossless drop-in for its LLM-based tool-result compression
|
|
41
|
+
from secondwind.agno import SecondwindCompressionManager
|
|
42
|
+
agent = Agent(model=..., compress_tool_results=True,
|
|
43
|
+
compression_manager=SecondwindCompressionManager(compress_tool_results=True))
|
|
44
|
+
|
|
45
|
+
# Strands Agents: compress tool results as they are produced
|
|
46
|
+
from secondwind.strands import SecondwindHooks
|
|
47
|
+
agent = Agent(model=..., tools=[...], hooks=[SecondwindHooks()])
|
|
48
|
+
|
|
49
|
+
# Cursor: a postToolUse hook that rewrites MCP tool output the model sees
|
|
50
|
+
# ~/.cursor/hooks.json -> { "version": 1, "hooks": { "postToolUse": [ { "command": "python -m secondwind.cursor" } ] } }
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
# Bring your own codec: it competes in the best-of-N search, and secondwind proves
|
|
55
|
+
# decode(encode(x)) == x for every block, so a wrong codec is dropped, never shipped.
|
|
56
|
+
class MyCodec:
|
|
57
|
+
def encode(self, text): ... # -> str | None
|
|
58
|
+
def decode(self, wire): ... # -> str | None
|
|
59
|
+
session = secondwind.Session(codec=MyCodec()) # reckless is safe: proven per-instance
|
|
60
|
+
session = secondwind.Session(proposers=False) # turn the aggressive search off
|
|
61
|
+
|
|
62
|
+
# Offload big blocks and recover them on demand, or back the store with Redis / S3 / a database.
|
|
63
|
+
session = secondwind.Session(resolver="resolve", offload_dir="~/.secondwind/offload")
|
|
64
|
+
session = secondwind.Session(store=MyStore()) # any object with put(id, value) and get(id)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Every result is lossless and independently verifiable (`secondwind.verify(wire, hash)`); the native
|
|
68
|
+
library is bundled, so there is no build step and no model download.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
secondwind/__init__.py,sha256=wicKKYZtwp47QiuGujJFSUJVealtTS0hqbxXvvtEgAs,8521
|
|
2
|
+
secondwind/agno.py,sha256=HtSqdJAc5j889mFGx_W4ebSFae6JNYbDKIdmzcq11cE,1919
|
|
3
|
+
secondwind/asgi.py,sha256=uKdOy3OQTBzs2f2TckVnvDAoSxhuEkaqK6FqJbwCkb8,3056
|
|
4
|
+
secondwind/cursor.py,sha256=-dwAqpNsKCSzpJI7boh7Mug072KHgneG3SwACq8wC-Y,2357
|
|
5
|
+
secondwind/langchain.py,sha256=XW1OCIJEGkSjQtxs9ttIs6bBbAK4e8TXZLfUfvd9qW4,3532
|
|
6
|
+
secondwind/litellm.py,sha256=eo4-hEXVqHishstFRsRxwlPVld5iRnQAllqsN6ewEho,1283
|
|
7
|
+
secondwind/strands.py,sha256=BIB4d8w0ZW7wWnDrVpB6HwMqKESqZ7usU8YSEe_W8t4,1858
|
|
8
|
+
secondwind/_lib/secondwind.dll,sha256=Gm4EkJsWY8Ss9v5nDI1EuXxvsSqcpA6aJ9VTl-ubSZk,25816064
|
|
9
|
+
secondwind-0.1.0.dist-info/METADATA,sha256=wcfxtAIHUIwF2G5m3AVYvD7YYFRPtweBRZ_E9vybNLE,2983
|
|
10
|
+
secondwind-0.1.0.dist-info/WHEEL,sha256=hVx9elvUDfBjRmbl8JwIcXXikst35RZTZK9nspfI_28,98
|
|
11
|
+
secondwind-0.1.0.dist-info/top_level.txt,sha256=5UNKECt3jgJKyrOzgZw9tb2VeJU-lz817ERGm-qNrAY,11
|
|
12
|
+
secondwind-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
secondwind
|