tokencut-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencut/__init__.py +0 -0
- tokencut/cli.py +127 -0
- tokencut/contracts.py +148 -0
- tokencut/gateway/__init__.py +0 -0
- tokencut/gateway/__main__.py +14 -0
- tokencut/gateway/adapters/__init__.py +1 -0
- tokencut/gateway/adapters/anthropic.py +119 -0
- tokencut/gateway/adapters/common.py +95 -0
- tokencut/gateway/adapters/gemini.py +107 -0
- tokencut/gateway/adapters/openai.py +82 -0
- tokencut/gateway/app.py +366 -0
- tokencut/gateway/brief.py +101 -0
- tokencut/gateway/ccr.py +83 -0
- tokencut/gateway/classify.py +41 -0
- tokencut/gateway/codeview.py +139 -0
- tokencut/gateway/history.py +433 -0
- tokencut/gateway/live.html +135 -0
- tokencut/gateway/media.py +825 -0
- tokencut/gateway/note.py +71 -0
- tokencut/gateway/prompt_router.py +214 -0
- tokencut/gateway/records.py +312 -0
- tokencut/gateway/routing.py +131 -0
- tokencut/gateway/session.py +76 -0
- tokencut/gateway/task_router.py +157 -0
- tokencut/gateway/textcut.py +91 -0
- tokencut/gateway/trimmer.py +256 -0
- tokencut/gateway/usage.py +120 -0
- tokencut/gateway/virtual.py +273 -0
- tokencut/graph/__init__.py +0 -0
- tokencut/graph/builder.py +307 -0
- tokencut/graph/modules.py +149 -0
- tokencut/graph/queries.py +106 -0
- tokencut/ingestion/__init__.py +0 -0
- tokencut/ingestion/base.py +18 -0
- tokencut/ingestion/parser.py +36 -0
- tokencut/ingestion/python_parser.py +275 -0
- tokencut/ingestion/walker.py +68 -0
- tokencut/launch.py +104 -0
- tokencut/litellm_proxy.py +134 -0
- tokencut/pdftool/__init__.py +9 -0
- tokencut/pdftool/__main__.py +115 -0
- tokencut/pdftool/reader.py +97 -0
- tokencut/repoindex/__init__.py +0 -0
- tokencut/repoindex/__main__.py +67 -0
- tokencut/repoindex/bm25.py +63 -0
- tokencut/repoindex/embed.py +93 -0
- tokencut/repoindex/index.py +455 -0
- tokencut/repoindex/mcp.py +145 -0
- tokencut/tokens.py +19 -0
- tokencut/trimming/__init__.py +0 -0
- tokencut/trimming/packing.py +317 -0
- tokencut/trimming/query.py +259 -0
- tokencut/trimming/render.py +65 -0
- tokencut/trimming/scoring.py +117 -0
- tokencut/trimming/variants.py +61 -0
- tokencut/userconfig.py +107 -0
- tokencut/wizard.py +94 -0
- tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
- tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
- tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
- tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
- tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
tokencut/__init__.py
ADDED
|
File without changes
|
tokencut/cli.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""The `tokencut` command (installed with the tokencut-proxy package).
|
|
2
|
+
|
|
3
|
+
tokencut [claude args...] open Claude Code through TokenCut (runs setup the first time)
|
|
4
|
+
tokencut setup logins and one-time installs (--chatgpt / --no-chatgpt)
|
|
5
|
+
tokencut status is it running, what did it save
|
|
6
|
+
tokencut logs [gateway|connector] [-n LINES]
|
|
7
|
+
tokencut gateway the gateway alone, in the foreground
|
|
8
|
+
tokencut --version
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
import urllib.request
|
|
16
|
+
from collections import deque
|
|
17
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
18
|
+
|
|
19
|
+
from tokencut import userconfig
|
|
20
|
+
|
|
21
|
+
COMMANDS = ("setup", "status", "logs", "gateway", "code")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _version() -> str:
|
|
25
|
+
try:
|
|
26
|
+
return version("tokencut-proxy")
|
|
27
|
+
except PackageNotFoundError: # running from a checkout without an install
|
|
28
|
+
return "0+checkout"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _gateway() -> int:
|
|
32
|
+
import uvicorn
|
|
33
|
+
|
|
34
|
+
from tokencut.gateway.app import create_app
|
|
35
|
+
|
|
36
|
+
uvicorn.run(create_app(), host=os.environ.get("TOKENCUT_HOST", "127.0.0.1"),
|
|
37
|
+
port=int(os.environ.get("TOKENCUT_PORT", "8787")))
|
|
38
|
+
return 0
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _get(url: str) -> dict | None:
|
|
42
|
+
try:
|
|
43
|
+
with urllib.request.urlopen(url, timeout=3) as response:
|
|
44
|
+
return json.loads(response.read())
|
|
45
|
+
except (OSError, ValueError):
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def status_text(health: dict | None, stats: dict | None, gateway: str) -> str:
|
|
50
|
+
if health is None:
|
|
51
|
+
return f"TokenCut is not running ({gateway}). Start it with `tokencut`."
|
|
52
|
+
features = [name for name in ("task_masking", "prune_tools", "ccr", "lazy_tools")
|
|
53
|
+
if health.get(name)]
|
|
54
|
+
models = "Claude + GPT" if health.get("gpt_upstream") else "Claude"
|
|
55
|
+
on = f"; on: {', '.join(features)}" if features else ""
|
|
56
|
+
lines = [f"TokenCut is running on {gateway} - mode {health['mode']}, tier {health['tier']}",
|
|
57
|
+
f" models: {models}{on}"]
|
|
58
|
+
if stats:
|
|
59
|
+
trimming = stats["trimming"]
|
|
60
|
+
saved = trimming["tokens_saved_applied"] or trimming["tokens_saved_possible"]
|
|
61
|
+
total = stats["tokens_total"]
|
|
62
|
+
share = f" ({saved / total:.0%} of {total:,})" if total else ""
|
|
63
|
+
billed = stats["billed_usage"]
|
|
64
|
+
lines.append(f" requests: {stats['requests']}, tokens removed: {saved:,}{share}")
|
|
65
|
+
lines.append(f" billed: input {billed['input_tokens']:,}, cache read "
|
|
66
|
+
f"{billed['cache_read_tokens']:,}, cache write "
|
|
67
|
+
f"{billed['cache_write_tokens']:,}, output {billed['output_tokens']:,}")
|
|
68
|
+
lines.append(f" live page: {gateway}/tokencut/")
|
|
69
|
+
return "\n".join(lines)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _status() -> int:
|
|
73
|
+
port = os.environ.get("TOKENCUT_PORT", str(userconfig.load()["gateway"]["port"]))
|
|
74
|
+
gateway = f"http://127.0.0.1:{port}"
|
|
75
|
+
health = _get(f"{gateway}/tokencut/health")
|
|
76
|
+
print(status_text(health, _get(f"{gateway}/tokencut/stats") if health else None, gateway))
|
|
77
|
+
return 0 if health else 1
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _logs(which: str, lines: int) -> int:
|
|
81
|
+
path = userconfig.home() / f"{which}.log"
|
|
82
|
+
if not path.exists():
|
|
83
|
+
print(f"no {which} log yet ({path})")
|
|
84
|
+
return 1
|
|
85
|
+
with path.open(encoding="utf-8", errors="replace") as handle:
|
|
86
|
+
sys.stdout.writelines(deque(handle, maxlen=lines))
|
|
87
|
+
return 0
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def main(argv: list[str] | None = None) -> int:
|
|
91
|
+
argv = sys.argv[1:] if argv is None else argv
|
|
92
|
+
if not argv or argv[0] == "code" or argv[0] not in (*COMMANDS, "-h", "--help", "--version"):
|
|
93
|
+
from tokencut import launch # every argument belongs to Claude Code
|
|
94
|
+
|
|
95
|
+
return launch.main(["code", *(argv[1:] if argv[:1] == ["code"] else argv)])
|
|
96
|
+
parser = argparse.ArgumentParser(prog="tokencut", description=__doc__.split("\n")[0])
|
|
97
|
+
parser.add_argument("--version", action="version", version=f"tokencut {_version()}")
|
|
98
|
+
commands = parser.add_subparsers(dest="command")
|
|
99
|
+
setup = commands.add_parser("setup", help="logins and one-time installs")
|
|
100
|
+
choice = setup.add_mutually_exclusive_group()
|
|
101
|
+
choice.add_argument("--chatgpt", action="store_true", default=None,
|
|
102
|
+
help="add the ChatGPT plan without asking")
|
|
103
|
+
choice.add_argument("--no-chatgpt", dest="chatgpt", action="store_false",
|
|
104
|
+
help="Claude only, without asking")
|
|
105
|
+
commands.add_parser("status", help="is it running, what did it save")
|
|
106
|
+
logs = commands.add_parser("logs", help="last lines of a process log")
|
|
107
|
+
logs.add_argument("which", nargs="?", default="gateway", choices=("gateway", "connector"))
|
|
108
|
+
logs.add_argument("-n", "--lines", type=int, default=40)
|
|
109
|
+
commands.add_parser("gateway", help="run the gateway in the foreground")
|
|
110
|
+
commands.add_parser("code", help="open Claude Code through TokenCut (the default)")
|
|
111
|
+
args = parser.parse_args(argv)
|
|
112
|
+
if args.command == "setup":
|
|
113
|
+
from tokencut.wizard import run_setup
|
|
114
|
+
|
|
115
|
+
return run_setup(chatgpt=args.chatgpt)
|
|
116
|
+
if args.command == "status":
|
|
117
|
+
return _status()
|
|
118
|
+
if args.command == "logs":
|
|
119
|
+
return _logs(args.which, args.lines)
|
|
120
|
+
if args.command == "gateway":
|
|
121
|
+
return _gateway()
|
|
122
|
+
parser.print_help()
|
|
123
|
+
return 2
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
if __name__ == "__main__":
|
|
127
|
+
sys.exit(main())
|
tokencut/contracts.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Shared data formats for Iteration 1.
|
|
2
|
+
|
|
3
|
+
Every module talks to the others ONLY through these classes:
|
|
4
|
+
|
|
5
|
+
ingestion.parse_repo() -> list[Chunk]
|
|
6
|
+
graph.build_graph(chunks) -> CodeGraph
|
|
7
|
+
trimming.generate_variants() -> list[ContextVariant]
|
|
8
|
+
|
|
9
|
+
RULE: do not change this file without a Pull Request approved by the other two members.
|
|
10
|
+
See docs/DECISIONS.md for the reasoning behind every rule below.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from enum import Enum
|
|
14
|
+
|
|
15
|
+
from pydantic import BaseModel, Field
|
|
16
|
+
|
|
17
|
+
# ---------- Shared settings (DECISIONS.md #4, #12) ----------
|
|
18
|
+
|
|
19
|
+
EVAL_FULL_CONTEXT_CAP_TOKENS = 32_000 # "full" context size in evaluation (tunable)
|
|
20
|
+
AGGRESSIVE_MIN_TOKENS = 1_000 # aggressive tier is never smaller than this
|
|
21
|
+
MAX_FILE_BYTES = 1_000_000 # files larger than this are skipped
|
|
22
|
+
SKIP_DIR_NAMES = {"migrations", "vendor", ".git", ".venv", "venv", "__pycache__", "node_modules"}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# ---------- Chunks (DECISIONS.md #2) ----------
|
|
26
|
+
|
|
27
|
+
class ChunkKind(str, Enum):
|
|
28
|
+
MODULE = "module" # module-level code: imports, constants, top-level statements (no defs)
|
|
29
|
+
CLASS = "class" # class header + class-level attributes ONLY (methods are separate chunks)
|
|
30
|
+
FUNCTION = "function" # top-level function (nested functions stay inside it)
|
|
31
|
+
METHOD = "method" # function defined directly inside a class
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Chunk(BaseModel):
|
|
35
|
+
"""One piece of code the trimmer can keep, skeletonize, or drop."""
|
|
36
|
+
|
|
37
|
+
id: str = Field(
|
|
38
|
+
description="'<file_path>::<qualified_name>', e.g. 'shop/cart.py::Cart.total'. "
|
|
39
|
+
"A repeated name in the same file gets a suffix: '...::Cart.price#2'"
|
|
40
|
+
)
|
|
41
|
+
file_path: str = Field(description="Relative to repo root, ALWAYS forward slashes")
|
|
42
|
+
qualified_name: str = Field(description="e.g. 'Cart.total'; '<module>' for module-level chunks")
|
|
43
|
+
kind: ChunkKind
|
|
44
|
+
parent_id: str | None = Field(default=None, description="For methods: id of their class chunk")
|
|
45
|
+
start_line: int = Field(ge=1, description="1-based, inclusive")
|
|
46
|
+
end_line: int = Field(ge=1, description="1-based, inclusive")
|
|
47
|
+
code: str
|
|
48
|
+
signature: str = Field(default="", description="e.g. 'def total(self) -> float:'; empty for module chunks")
|
|
49
|
+
docstring: str = ""
|
|
50
|
+
token_count: int = Field(ge=0, description="Counted with tokencut.tokens.count_tokens")
|
|
51
|
+
file_hash: str = Field(description="sha256 of the whole file, for caching / incremental updates")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# ---------- Graph (DECISIONS.md #7) ----------
|
|
55
|
+
|
|
56
|
+
class EdgeKind(str, Enum):
|
|
57
|
+
CONTAINS = "contains" # class -> method
|
|
58
|
+
IMPORTS = "imports" # module chunk -> module chunk of the imported file
|
|
59
|
+
CALLS = "calls" # function/method -> function/method it calls
|
|
60
|
+
INHERITS = "inherits" # class -> base class
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Edge(BaseModel):
|
|
64
|
+
source: str = Field(description="Chunk id")
|
|
65
|
+
target: str = Field(description="Chunk id (must exist in the graph)")
|
|
66
|
+
kind: EdgeKind
|
|
67
|
+
resolved: bool = Field(default=True, description="False if found by unique-name guess (rule 4)")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class CodeGraph(BaseModel):
|
|
71
|
+
chunk_ids: list[str]
|
|
72
|
+
edges: list[Edge]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# ---------- Tiers and variants (DECISIONS.md #4, #5) ----------
|
|
76
|
+
|
|
77
|
+
class Tier(str, Enum):
|
|
78
|
+
AGGRESSIVE = "aggressive"
|
|
79
|
+
MODERATE = "moderate"
|
|
80
|
+
CONSERVATIVE = "conservative"
|
|
81
|
+
FULL = "full"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# Fraction of the "full" context's tokens per tier (tunable).
|
|
85
|
+
TIER_BUDGET_FRACTION: dict[Tier, float] = {
|
|
86
|
+
Tier.AGGRESSIVE: 0.15,
|
|
87
|
+
Tier.MODERATE: 0.35,
|
|
88
|
+
Tier.CONSERVATIVE: 0.60,
|
|
89
|
+
Tier.FULL: 1.00,
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
# Tiers in which connected dropped chunks are kept as skeletons (signature + docstring).
|
|
93
|
+
SKELETON_TIERS: set[Tier] = {Tier.AGGRESSIVE, Tier.MODERATE}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class ScoredChunk(BaseModel):
|
|
97
|
+
chunk_id: str
|
|
98
|
+
score: float = Field(description="Final relevance score, higher = more relevant")
|
|
99
|
+
structural_score: float = 0.0
|
|
100
|
+
text_score: float = 0.0
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class ContextVariant(BaseModel):
|
|
104
|
+
tier: Tier
|
|
105
|
+
budget_tokens: int
|
|
106
|
+
kept_chunk_ids: list[str] = Field(description="Full code kept; sorted by file path, then start line")
|
|
107
|
+
skeleton_chunk_ids: list[str] = Field(
|
|
108
|
+
default_factory=list, description="Kept as signature + docstring only; same sort order"
|
|
109
|
+
)
|
|
110
|
+
token_count: int = Field(description="Includes skeleton tokens; must be <= budget_tokens")
|
|
111
|
+
rendered_text: str = Field(description="The text that would be sent to the model")
|
|
112
|
+
|
|
113
|
+
# ---------- Gateway requests (Iteration 2) ----------
|
|
114
|
+
# A provider request (Anthropic / OpenAI / Gemini) is split into Segments by a provider
|
|
115
|
+
# adapter. Trimming and routing only ever see Segments, so they are written once for all
|
|
116
|
+
# providers. `path` locates the segment's text inside the original JSON body, so an adapter
|
|
117
|
+
# can write shortened text back without understanding the trimming.
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class SegmentKind(str, Enum):
|
|
121
|
+
SYSTEM = "system" # system prompt / instructions (never trimmed)
|
|
122
|
+
TOOLS = "tools" # tool definitions (never trimmed)
|
|
123
|
+
USER_TEXT = "user_text"
|
|
124
|
+
ASSISTANT_TEXT = "assistant_text"
|
|
125
|
+
THINKING = "thinking" # model reasoning blocks (never trimmed)
|
|
126
|
+
TOOL_CALL = "tool_call" # assistant asks to run a tool (never trimmed)
|
|
127
|
+
TOOL_RESULT = "tool_result" # output of a tool: file contents, command output, ...
|
|
128
|
+
MEDIA = "media" # images, documents, unknown block types: never trimmed
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class ToolResultKind(str, Enum):
|
|
132
|
+
FILE_READ = "file_read"
|
|
133
|
+
COMMAND_OUTPUT = "command_output"
|
|
134
|
+
SEARCH = "search"
|
|
135
|
+
OTHER = "other"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class Segment(BaseModel):
|
|
139
|
+
id: str = Field(description="Unique within one request, e.g. 'm3.b1' (message 3, block 1)")
|
|
140
|
+
kind: SegmentKind
|
|
141
|
+
role: str = Field(description="Provider role: system / user / assistant / tool / model")
|
|
142
|
+
message_index: int = Field(ge=-1, description="Index in the message list; -1 = outside it")
|
|
143
|
+
path: list[str | int] = Field(description="JSON path to this segment's text in the body")
|
|
144
|
+
text: str = ""
|
|
145
|
+
tool_name: str = ""
|
|
146
|
+
tool_call_id: str = ""
|
|
147
|
+
result_kind: ToolResultKind | None = None
|
|
148
|
+
token_count: int = Field(ge=0)
|
|
File without changes
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Run the gateway: python -m tokencut.gateway (listens on 127.0.0.1:8787 by default)."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
import uvicorn
|
|
6
|
+
|
|
7
|
+
from tokencut.gateway.app import create_app
|
|
8
|
+
|
|
9
|
+
if __name__ == "__main__":
|
|
10
|
+
uvicorn.run(
|
|
11
|
+
create_app(),
|
|
12
|
+
host=os.environ.get("TOKENCUT_HOST", "127.0.0.1"), # local only unless asked otherwise
|
|
13
|
+
port=int(os.environ.get("TOKENCUT_PORT", "8787")),
|
|
14
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Provider adapters: turn a provider's request JSON into list[Segment] (and later back)."""
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Anthropic Messages API (POST /v1/messages): Claude Code and other Claude clients."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from tokencut.contracts import Segment, SegmentKind
|
|
6
|
+
from tokencut.gateway.adapters.common import SegmentBuilder, as_json
|
|
7
|
+
|
|
8
|
+
PROVIDER = "anthropic"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def model_of(body: dict, path: str) -> str:
|
|
12
|
+
return str(body.get("model", ""))
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def parse(body: dict) -> list[Segment]:
|
|
16
|
+
b = SegmentBuilder()
|
|
17
|
+
|
|
18
|
+
system = body.get("system")
|
|
19
|
+
if isinstance(system, str):
|
|
20
|
+
b.add("system", SegmentKind.SYSTEM, "system", -1, ["system"], system)
|
|
21
|
+
elif isinstance(system, list):
|
|
22
|
+
for j, block in enumerate(system):
|
|
23
|
+
if isinstance(block, dict):
|
|
24
|
+
b.add(
|
|
25
|
+
f"system.{j}",
|
|
26
|
+
SegmentKind.SYSTEM,
|
|
27
|
+
"system",
|
|
28
|
+
-1,
|
|
29
|
+
["system", j, "text"],
|
|
30
|
+
str(block.get("text", "")),
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
if body.get("tools"):
|
|
34
|
+
b.add("tools", SegmentKind.TOOLS, "system", -1, ["tools"], as_json(body["tools"]))
|
|
35
|
+
|
|
36
|
+
tool_names: dict[str, str] = {} # tool_use id -> tool name (results only carry the id)
|
|
37
|
+
for i, message in enumerate(body.get("messages", [])):
|
|
38
|
+
role = str(message.get("role", ""))
|
|
39
|
+
# Claude Code sends its environment / skills list as role "system" messages: agent
|
|
40
|
+
# context, not the user's words (as role "system" / "developer" in the OpenAI adapter).
|
|
41
|
+
text_kind = {"assistant": SegmentKind.ASSISTANT_TEXT,
|
|
42
|
+
"system": SegmentKind.SYSTEM}.get(role, SegmentKind.USER_TEXT)
|
|
43
|
+
content = message.get("content")
|
|
44
|
+
if isinstance(content, str):
|
|
45
|
+
b.add(f"m{i}", text_kind, role, i, ["messages", i, "content"], content)
|
|
46
|
+
continue
|
|
47
|
+
if not isinstance(content, list):
|
|
48
|
+
continue
|
|
49
|
+
for j, block in enumerate(content):
|
|
50
|
+
_parse_block(b, block, role, i, j, text_kind, tool_names)
|
|
51
|
+
return b.segments
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _parse_block(
|
|
55
|
+
b: SegmentBuilder,
|
|
56
|
+
block: Any,
|
|
57
|
+
role: str,
|
|
58
|
+
i: int,
|
|
59
|
+
j: int,
|
|
60
|
+
text_kind: SegmentKind,
|
|
61
|
+
tool_names: dict[str, str],
|
|
62
|
+
) -> None:
|
|
63
|
+
if not isinstance(block, dict):
|
|
64
|
+
return
|
|
65
|
+
seg_id = f"m{i}.b{j}"
|
|
66
|
+
path: list[str | int] = ["messages", i, "content", j]
|
|
67
|
+
block_type = block.get("type")
|
|
68
|
+
if block_type == "text":
|
|
69
|
+
b.add(seg_id, text_kind, role, i, path + ["text"], str(block.get("text", "")))
|
|
70
|
+
elif block_type == "thinking":
|
|
71
|
+
b.add(
|
|
72
|
+
seg_id,
|
|
73
|
+
SegmentKind.THINKING,
|
|
74
|
+
role,
|
|
75
|
+
i,
|
|
76
|
+
path + ["thinking"],
|
|
77
|
+
str(block.get("thinking", "")),
|
|
78
|
+
)
|
|
79
|
+
elif block_type == "redacted_thinking":
|
|
80
|
+
b.add(seg_id, SegmentKind.THINKING, role, i, path, "")
|
|
81
|
+
elif block_type == "tool_use":
|
|
82
|
+
call_id = str(block.get("id", ""))
|
|
83
|
+
name = str(block.get("name", ""))
|
|
84
|
+
tool_names[call_id] = name
|
|
85
|
+
b.add(
|
|
86
|
+
seg_id,
|
|
87
|
+
SegmentKind.TOOL_CALL,
|
|
88
|
+
role,
|
|
89
|
+
i,
|
|
90
|
+
path + ["input"],
|
|
91
|
+
as_json(block.get("input", {})),
|
|
92
|
+
name,
|
|
93
|
+
call_id,
|
|
94
|
+
)
|
|
95
|
+
elif block_type == "tool_result":
|
|
96
|
+
call_id = str(block.get("tool_use_id", ""))
|
|
97
|
+
name = tool_names.get(call_id, "")
|
|
98
|
+
inner = block.get("content", "")
|
|
99
|
+
if isinstance(inner, str):
|
|
100
|
+
b.add_tool_result(seg_id, role, i, path + ["content"], inner, name, call_id)
|
|
101
|
+
elif isinstance(inner, list):
|
|
102
|
+
for k, sub in enumerate(inner):
|
|
103
|
+
sub_id = f"{seg_id}.c{k}"
|
|
104
|
+
if isinstance(sub, dict) and sub.get("type") == "text":
|
|
105
|
+
b.add_tool_result(
|
|
106
|
+
sub_id,
|
|
107
|
+
role,
|
|
108
|
+
i,
|
|
109
|
+
path + ["content", k, "text"],
|
|
110
|
+
str(sub.get("text", "")),
|
|
111
|
+
name,
|
|
112
|
+
call_id,
|
|
113
|
+
)
|
|
114
|
+
else:
|
|
115
|
+
b.add(sub_id, SegmentKind.MEDIA, role, i, path + ["content", k], "")
|
|
116
|
+
else:
|
|
117
|
+
# image, document, server tool blocks, future block types: counted, never trimmed
|
|
118
|
+
text = "" if block_type in ("image", "document") else as_json(block)
|
|
119
|
+
b.add(seg_id, SegmentKind.MEDIA, role, i, path, text)
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Helpers shared by the provider adapters."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from tokencut.contracts import Segment, SegmentKind, ToolResultKind
|
|
7
|
+
from tokencut.gateway.classify import classify_tool_result
|
|
8
|
+
from tokencut.tokens import count_tokens
|
|
9
|
+
|
|
10
|
+
JsonPath = list[str | int]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def as_json(value: Any) -> str:
|
|
14
|
+
"""Stable text form of a JSON value, for token counting."""
|
|
15
|
+
return json.dumps(value, ensure_ascii=False, sort_keys=True)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get(obj: dict, camel: str, snake: str, default: Any = None) -> Any:
|
|
19
|
+
"""Gemini's REST API accepts both camelCase and snake_case field names."""
|
|
20
|
+
if camel in obj:
|
|
21
|
+
return obj[camel]
|
|
22
|
+
return obj.get(snake, default)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def key_name(obj: dict, camel: str, snake: str) -> str:
|
|
26
|
+
return camel if camel in obj else snake
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SegmentBuilder:
|
|
30
|
+
"""Collects segments with unique ids and token counts."""
|
|
31
|
+
|
|
32
|
+
def __init__(self) -> None:
|
|
33
|
+
self.segments: list[Segment] = []
|
|
34
|
+
|
|
35
|
+
def add(
|
|
36
|
+
self,
|
|
37
|
+
seg_id: str,
|
|
38
|
+
kind: SegmentKind,
|
|
39
|
+
role: str,
|
|
40
|
+
message_index: int,
|
|
41
|
+
path: JsonPath,
|
|
42
|
+
text: str,
|
|
43
|
+
tool_name: str = "",
|
|
44
|
+
tool_call_id: str = "",
|
|
45
|
+
result_kind: ToolResultKind | None = None,
|
|
46
|
+
) -> None:
|
|
47
|
+
self.segments.append(
|
|
48
|
+
Segment(
|
|
49
|
+
id=seg_id,
|
|
50
|
+
kind=kind,
|
|
51
|
+
role=role,
|
|
52
|
+
message_index=message_index,
|
|
53
|
+
path=list(path),
|
|
54
|
+
text=text,
|
|
55
|
+
tool_name=tool_name,
|
|
56
|
+
tool_call_id=tool_call_id,
|
|
57
|
+
result_kind=result_kind,
|
|
58
|
+
token_count=count_tokens(text),
|
|
59
|
+
)
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
def add_tool_result(
|
|
63
|
+
self,
|
|
64
|
+
seg_id: str,
|
|
65
|
+
role: str,
|
|
66
|
+
message_index: int,
|
|
67
|
+
path: JsonPath,
|
|
68
|
+
text: str,
|
|
69
|
+
tool_name: str,
|
|
70
|
+
tool_call_id: str,
|
|
71
|
+
) -> None:
|
|
72
|
+
self.add(
|
|
73
|
+
seg_id,
|
|
74
|
+
SegmentKind.TOOL_RESULT,
|
|
75
|
+
role,
|
|
76
|
+
message_index,
|
|
77
|
+
path,
|
|
78
|
+
text,
|
|
79
|
+
tool_name,
|
|
80
|
+
tool_call_id,
|
|
81
|
+
classify_tool_result(tool_name, text),
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def lookup(body: Any, path: JsonPath) -> Any:
|
|
86
|
+
value = body
|
|
87
|
+
for key in path:
|
|
88
|
+
value = value[key]
|
|
89
|
+
return value
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def set_text(body: Any, path: JsonPath, text: str) -> None:
|
|
93
|
+
"""Replace the string at `path` (the location a Segment points to)."""
|
|
94
|
+
parent = lookup(body, path[:-1])
|
|
95
|
+
parent[path[-1]] = text
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Google Gemini API (POST /v1beta/models/{model}:generateContent / :streamGenerateContent).
|
|
2
|
+
|
|
3
|
+
Gemini CLI and google-genai clients. The REST API accepts camelCase and snake_case keys.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
from tokencut.contracts import Segment, SegmentKind
|
|
9
|
+
from tokencut.gateway.adapters.common import SegmentBuilder, as_json, get, key_name
|
|
10
|
+
|
|
11
|
+
PROVIDER = "gemini"
|
|
12
|
+
_MODEL_IN_PATH = re.compile(r"models/([^/:]+):")
|
|
13
|
+
# Common single-string fields in functionResponse.response (Gemini CLI uses "output").
|
|
14
|
+
_RESULT_TEXT_FIELDS = ("output", "content", "result", "text")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def model_of(body: dict, path: str) -> str:
|
|
18
|
+
match = _MODEL_IN_PATH.search(path)
|
|
19
|
+
return match.group(1) if match else ""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def parse(body: dict) -> list[Segment]:
|
|
23
|
+
b = SegmentBuilder()
|
|
24
|
+
system_key = key_name(body, "systemInstruction", "system_instruction")
|
|
25
|
+
system = body.get(system_key)
|
|
26
|
+
if isinstance(system, dict):
|
|
27
|
+
for j, part in enumerate(system.get("parts", [])):
|
|
28
|
+
if isinstance(part, dict) and "text" in part:
|
|
29
|
+
b.add(
|
|
30
|
+
f"system.{j}",
|
|
31
|
+
SegmentKind.SYSTEM,
|
|
32
|
+
"system",
|
|
33
|
+
-1,
|
|
34
|
+
[system_key, "parts", j, "text"],
|
|
35
|
+
str(part["text"]),
|
|
36
|
+
)
|
|
37
|
+
if body.get("tools"):
|
|
38
|
+
b.add("tools", SegmentKind.TOOLS, "system", -1, ["tools"], as_json(body["tools"]))
|
|
39
|
+
|
|
40
|
+
for i, content in enumerate(body.get("contents", [])):
|
|
41
|
+
role = str(content.get("role", "user"))
|
|
42
|
+
text_kind = SegmentKind.ASSISTANT_TEXT if role == "model" else SegmentKind.USER_TEXT
|
|
43
|
+
for j, part in enumerate(content.get("parts", [])):
|
|
44
|
+
if not isinstance(part, dict):
|
|
45
|
+
continue
|
|
46
|
+
seg_id = f"m{i}.p{j}"
|
|
47
|
+
path: list[str | int] = ["contents", i, "parts", j]
|
|
48
|
+
call = get(part, "functionCall", "function_call")
|
|
49
|
+
response = get(part, "functionResponse", "function_response")
|
|
50
|
+
if "text" in part:
|
|
51
|
+
kind = SegmentKind.THINKING if part.get("thought") else text_kind
|
|
52
|
+
b.add(seg_id, kind, role, i, path + ["text"], str(part["text"]))
|
|
53
|
+
elif isinstance(call, dict):
|
|
54
|
+
b.add(
|
|
55
|
+
seg_id,
|
|
56
|
+
SegmentKind.TOOL_CALL,
|
|
57
|
+
role,
|
|
58
|
+
i,
|
|
59
|
+
path + [key_name(part, "functionCall", "function_call"), "args"],
|
|
60
|
+
as_json(call.get("args", {})),
|
|
61
|
+
str(call.get("name", "")),
|
|
62
|
+
str(call.get("id", "")),
|
|
63
|
+
)
|
|
64
|
+
elif isinstance(response, dict):
|
|
65
|
+
fr_key = key_name(part, "functionResponse", "function_response")
|
|
66
|
+
name = str(response.get("name", ""))
|
|
67
|
+
call_id = str(response.get("id", ""))
|
|
68
|
+
payload = response.get("response", {})
|
|
69
|
+
field = _text_field(payload)
|
|
70
|
+
if field is not None:
|
|
71
|
+
b.add_tool_result(
|
|
72
|
+
seg_id,
|
|
73
|
+
role,
|
|
74
|
+
i,
|
|
75
|
+
path + [fr_key, "response", field],
|
|
76
|
+
payload[field],
|
|
77
|
+
name,
|
|
78
|
+
call_id,
|
|
79
|
+
)
|
|
80
|
+
else:
|
|
81
|
+
b.add_tool_result(
|
|
82
|
+
seg_id,
|
|
83
|
+
role,
|
|
84
|
+
i,
|
|
85
|
+
path + [fr_key, "response"],
|
|
86
|
+
as_json(payload),
|
|
87
|
+
name,
|
|
88
|
+
call_id,
|
|
89
|
+
)
|
|
90
|
+
# Gemini CLI nests a tool's images / documents here on multimodal models
|
|
91
|
+
for m, nested in enumerate(response.get("parts") or []):
|
|
92
|
+
if isinstance(nested, dict):
|
|
93
|
+
b.add(f"{seg_id}.n{m}", SegmentKind.MEDIA, role, i,
|
|
94
|
+
path + [fr_key, "parts", m], "")
|
|
95
|
+
else: # inlineData, fileData, executableCode, ...
|
|
96
|
+
b.add(seg_id, SegmentKind.MEDIA, role, i, path, "")
|
|
97
|
+
return b.segments
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _text_field(payload: object) -> str | None:
|
|
101
|
+
"""The one string field holding the tool output, if the response has that shape."""
|
|
102
|
+
if not isinstance(payload, dict):
|
|
103
|
+
return None
|
|
104
|
+
for field in _RESULT_TEXT_FIELDS:
|
|
105
|
+
if isinstance(payload.get(field), str) and len(payload) <= 2:
|
|
106
|
+
return field
|
|
107
|
+
return None
|