tokencut-proxy 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. tokencut/__init__.py +0 -0
  2. tokencut/cli.py +127 -0
  3. tokencut/contracts.py +148 -0
  4. tokencut/gateway/__init__.py +0 -0
  5. tokencut/gateway/__main__.py +14 -0
  6. tokencut/gateway/adapters/__init__.py +1 -0
  7. tokencut/gateway/adapters/anthropic.py +119 -0
  8. tokencut/gateway/adapters/common.py +95 -0
  9. tokencut/gateway/adapters/gemini.py +107 -0
  10. tokencut/gateway/adapters/openai.py +82 -0
  11. tokencut/gateway/app.py +366 -0
  12. tokencut/gateway/brief.py +101 -0
  13. tokencut/gateway/ccr.py +83 -0
  14. tokencut/gateway/classify.py +41 -0
  15. tokencut/gateway/codeview.py +139 -0
  16. tokencut/gateway/history.py +433 -0
  17. tokencut/gateway/live.html +135 -0
  18. tokencut/gateway/media.py +825 -0
  19. tokencut/gateway/note.py +71 -0
  20. tokencut/gateway/prompt_router.py +214 -0
  21. tokencut/gateway/records.py +312 -0
  22. tokencut/gateway/routing.py +131 -0
  23. tokencut/gateway/session.py +76 -0
  24. tokencut/gateway/task_router.py +157 -0
  25. tokencut/gateway/textcut.py +91 -0
  26. tokencut/gateway/trimmer.py +256 -0
  27. tokencut/gateway/usage.py +120 -0
  28. tokencut/gateway/virtual.py +273 -0
  29. tokencut/graph/__init__.py +0 -0
  30. tokencut/graph/builder.py +307 -0
  31. tokencut/graph/modules.py +149 -0
  32. tokencut/graph/queries.py +106 -0
  33. tokencut/ingestion/__init__.py +0 -0
  34. tokencut/ingestion/base.py +18 -0
  35. tokencut/ingestion/parser.py +36 -0
  36. tokencut/ingestion/python_parser.py +275 -0
  37. tokencut/ingestion/walker.py +68 -0
  38. tokencut/launch.py +104 -0
  39. tokencut/litellm_proxy.py +134 -0
  40. tokencut/pdftool/__init__.py +9 -0
  41. tokencut/pdftool/__main__.py +115 -0
  42. tokencut/pdftool/reader.py +97 -0
  43. tokencut/repoindex/__init__.py +0 -0
  44. tokencut/repoindex/__main__.py +67 -0
  45. tokencut/repoindex/bm25.py +63 -0
  46. tokencut/repoindex/embed.py +93 -0
  47. tokencut/repoindex/index.py +455 -0
  48. tokencut/repoindex/mcp.py +145 -0
  49. tokencut/tokens.py +19 -0
  50. tokencut/trimming/__init__.py +0 -0
  51. tokencut/trimming/packing.py +317 -0
  52. tokencut/trimming/query.py +259 -0
  53. tokencut/trimming/render.py +65 -0
  54. tokencut/trimming/scoring.py +117 -0
  55. tokencut/trimming/variants.py +61 -0
  56. tokencut/userconfig.py +107 -0
  57. tokencut/wizard.py +94 -0
  58. tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
  59. tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
  60. tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
  61. tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
  62. tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
tokencut/__init__.py ADDED
File without changes
tokencut/cli.py ADDED
@@ -0,0 +1,127 @@
1
+ """The `tokencut` command (installed with the tokencut-proxy package).
2
+
3
+ tokencut [claude args...] open Claude Code through TokenCut (runs setup the first time)
4
+ tokencut setup logins and one-time installs (--chatgpt / --no-chatgpt)
5
+ tokencut status is it running, what did it save
6
+ tokencut logs [gateway|connector] [-n LINES]
7
+ tokencut gateway the gateway alone, in the foreground
8
+ tokencut --version
9
+ """
10
+
11
+ import argparse
12
+ import json
13
+ import os
14
+ import sys
15
+ import urllib.request
16
+ from collections import deque
17
+ from importlib.metadata import PackageNotFoundError, version
18
+
19
+ from tokencut import userconfig
20
+
21
+ COMMANDS = ("setup", "status", "logs", "gateway", "code")
22
+
23
+
24
+ def _version() -> str:
25
+ try:
26
+ return version("tokencut-proxy")
27
+ except PackageNotFoundError: # running from a checkout without an install
28
+ return "0+checkout"
29
+
30
+
31
+ def _gateway() -> int:
32
+ import uvicorn
33
+
34
+ from tokencut.gateway.app import create_app
35
+
36
+ uvicorn.run(create_app(), host=os.environ.get("TOKENCUT_HOST", "127.0.0.1"),
37
+ port=int(os.environ.get("TOKENCUT_PORT", "8787")))
38
+ return 0
39
+
40
+
41
+ def _get(url: str) -> dict | None:
42
+ try:
43
+ with urllib.request.urlopen(url, timeout=3) as response:
44
+ return json.loads(response.read())
45
+ except (OSError, ValueError):
46
+ return None
47
+
48
+
49
+ def status_text(health: dict | None, stats: dict | None, gateway: str) -> str:
50
+ if health is None:
51
+ return f"TokenCut is not running ({gateway}). Start it with `tokencut`."
52
+ features = [name for name in ("task_masking", "prune_tools", "ccr", "lazy_tools")
53
+ if health.get(name)]
54
+ models = "Claude + GPT" if health.get("gpt_upstream") else "Claude"
55
+ on = f"; on: {', '.join(features)}" if features else ""
56
+ lines = [f"TokenCut is running on {gateway} - mode {health['mode']}, tier {health['tier']}",
57
+ f" models: {models}{on}"]
58
+ if stats:
59
+ trimming = stats["trimming"]
60
+ saved = trimming["tokens_saved_applied"] or trimming["tokens_saved_possible"]
61
+ total = stats["tokens_total"]
62
+ share = f" ({saved / total:.0%} of {total:,})" if total else ""
63
+ billed = stats["billed_usage"]
64
+ lines.append(f" requests: {stats['requests']}, tokens removed: {saved:,}{share}")
65
+ lines.append(f" billed: input {billed['input_tokens']:,}, cache read "
66
+ f"{billed['cache_read_tokens']:,}, cache write "
67
+ f"{billed['cache_write_tokens']:,}, output {billed['output_tokens']:,}")
68
+ lines.append(f" live page: {gateway}/tokencut/")
69
+ return "\n".join(lines)
70
+
71
+
72
+ def _status() -> int:
73
+ port = os.environ.get("TOKENCUT_PORT", str(userconfig.load()["gateway"]["port"]))
74
+ gateway = f"http://127.0.0.1:{port}"
75
+ health = _get(f"{gateway}/tokencut/health")
76
+ print(status_text(health, _get(f"{gateway}/tokencut/stats") if health else None, gateway))
77
+ return 0 if health else 1
78
+
79
+
80
+ def _logs(which: str, lines: int) -> int:
81
+ path = userconfig.home() / f"{which}.log"
82
+ if not path.exists():
83
+ print(f"no {which} log yet ({path})")
84
+ return 1
85
+ with path.open(encoding="utf-8", errors="replace") as handle:
86
+ sys.stdout.writelines(deque(handle, maxlen=lines))
87
+ return 0
88
+
89
+
90
+ def main(argv: list[str] | None = None) -> int:
91
+ argv = sys.argv[1:] if argv is None else argv
92
+ if not argv or argv[0] == "code" or argv[0] not in (*COMMANDS, "-h", "--help", "--version"):
93
+ from tokencut import launch # every argument belongs to Claude Code
94
+
95
+ return launch.main(["code", *(argv[1:] if argv[:1] == ["code"] else argv)])
96
+ parser = argparse.ArgumentParser(prog="tokencut", description=__doc__.split("\n")[0])
97
+ parser.add_argument("--version", action="version", version=f"tokencut {_version()}")
98
+ commands = parser.add_subparsers(dest="command")
99
+ setup = commands.add_parser("setup", help="logins and one-time installs")
100
+ choice = setup.add_mutually_exclusive_group()
101
+ choice.add_argument("--chatgpt", action="store_true", default=None,
102
+ help="add the ChatGPT plan without asking")
103
+ choice.add_argument("--no-chatgpt", dest="chatgpt", action="store_false",
104
+ help="Claude only, without asking")
105
+ commands.add_parser("status", help="is it running, what did it save")
106
+ logs = commands.add_parser("logs", help="last lines of a process log")
107
+ logs.add_argument("which", nargs="?", default="gateway", choices=("gateway", "connector"))
108
+ logs.add_argument("-n", "--lines", type=int, default=40)
109
+ commands.add_parser("gateway", help="run the gateway in the foreground")
110
+ commands.add_parser("code", help="open Claude Code through TokenCut (the default)")
111
+ args = parser.parse_args(argv)
112
+ if args.command == "setup":
113
+ from tokencut.wizard import run_setup
114
+
115
+ return run_setup(chatgpt=args.chatgpt)
116
+ if args.command == "status":
117
+ return _status()
118
+ if args.command == "logs":
119
+ return _logs(args.which, args.lines)
120
+ if args.command == "gateway":
121
+ return _gateway()
122
+ parser.print_help()
123
+ return 2
124
+
125
+
126
+ if __name__ == "__main__":
127
+ sys.exit(main())
tokencut/contracts.py ADDED
@@ -0,0 +1,148 @@
1
+ """Shared data formats for Iteration 1.
2
+
3
+ Every module talks to the others ONLY through these classes:
4
+
5
+ ingestion.parse_repo() -> list[Chunk]
6
+ graph.build_graph(chunks) -> CodeGraph
7
+ trimming.generate_variants() -> list[ContextVariant]
8
+
9
+ RULE: do not change this file without a Pull Request approved by the other two members.
10
+ See docs/DECISIONS.md for the reasoning behind every rule below.
11
+ """
12
+
13
+ from enum import Enum
14
+
15
+ from pydantic import BaseModel, Field
16
+
17
+ # ---------- Shared settings (DECISIONS.md #4, #12) ----------
18
+
19
+ EVAL_FULL_CONTEXT_CAP_TOKENS = 32_000 # "full" context size in evaluation (tunable)
20
+ AGGRESSIVE_MIN_TOKENS = 1_000 # aggressive tier is never smaller than this
21
+ MAX_FILE_BYTES = 1_000_000 # files larger than this are skipped
22
+ SKIP_DIR_NAMES = {"migrations", "vendor", ".git", ".venv", "venv", "__pycache__", "node_modules"}
23
+
24
+
25
+ # ---------- Chunks (DECISIONS.md #2) ----------
26
+
27
+ class ChunkKind(str, Enum):
28
+ MODULE = "module" # module-level code: imports, constants, top-level statements (no defs)
29
+ CLASS = "class" # class header + class-level attributes ONLY (methods are separate chunks)
30
+ FUNCTION = "function" # top-level function (nested functions stay inside it)
31
+ METHOD = "method" # function defined directly inside a class
32
+
33
+
34
+ class Chunk(BaseModel):
35
+ """One piece of code the trimmer can keep, skeletonize, or drop."""
36
+
37
+ id: str = Field(
38
+ description="'<file_path>::<qualified_name>', e.g. 'shop/cart.py::Cart.total'. "
39
+ "A repeated name in the same file gets a suffix: '...::Cart.price#2'"
40
+ )
41
+ file_path: str = Field(description="Relative to repo root, ALWAYS forward slashes")
42
+ qualified_name: str = Field(description="e.g. 'Cart.total'; '<module>' for module-level chunks")
43
+ kind: ChunkKind
44
+ parent_id: str | None = Field(default=None, description="For methods: id of their class chunk")
45
+ start_line: int = Field(ge=1, description="1-based, inclusive")
46
+ end_line: int = Field(ge=1, description="1-based, inclusive")
47
+ code: str
48
+ signature: str = Field(default="", description="e.g. 'def total(self) -> float:'; empty for module chunks")
49
+ docstring: str = ""
50
+ token_count: int = Field(ge=0, description="Counted with tokencut.tokens.count_tokens")
51
+ file_hash: str = Field(description="sha256 of the whole file, for caching / incremental updates")
52
+
53
+
54
+ # ---------- Graph (DECISIONS.md #7) ----------
55
+
56
+ class EdgeKind(str, Enum):
57
+ CONTAINS = "contains" # class -> method
58
+ IMPORTS = "imports" # module chunk -> module chunk of the imported file
59
+ CALLS = "calls" # function/method -> function/method it calls
60
+ INHERITS = "inherits" # class -> base class
61
+
62
+
63
+ class Edge(BaseModel):
64
+ source: str = Field(description="Chunk id")
65
+ target: str = Field(description="Chunk id (must exist in the graph)")
66
+ kind: EdgeKind
67
+ resolved: bool = Field(default=True, description="False if found by unique-name guess (rule 4)")
68
+
69
+
70
+ class CodeGraph(BaseModel):
71
+ chunk_ids: list[str]
72
+ edges: list[Edge]
73
+
74
+
75
+ # ---------- Tiers and variants (DECISIONS.md #4, #5) ----------
76
+
77
+ class Tier(str, Enum):
78
+ AGGRESSIVE = "aggressive"
79
+ MODERATE = "moderate"
80
+ CONSERVATIVE = "conservative"
81
+ FULL = "full"
82
+
83
+
84
+ # Fraction of the "full" context's tokens per tier (tunable).
85
+ TIER_BUDGET_FRACTION: dict[Tier, float] = {
86
+ Tier.AGGRESSIVE: 0.15,
87
+ Tier.MODERATE: 0.35,
88
+ Tier.CONSERVATIVE: 0.60,
89
+ Tier.FULL: 1.00,
90
+ }
91
+
92
+ # Tiers in which connected dropped chunks are kept as skeletons (signature + docstring).
93
+ SKELETON_TIERS: set[Tier] = {Tier.AGGRESSIVE, Tier.MODERATE}
94
+
95
+
96
+ class ScoredChunk(BaseModel):
97
+ chunk_id: str
98
+ score: float = Field(description="Final relevance score, higher = more relevant")
99
+ structural_score: float = 0.0
100
+ text_score: float = 0.0
101
+
102
+
103
+ class ContextVariant(BaseModel):
104
+ tier: Tier
105
+ budget_tokens: int
106
+ kept_chunk_ids: list[str] = Field(description="Full code kept; sorted by file path, then start line")
107
+ skeleton_chunk_ids: list[str] = Field(
108
+ default_factory=list, description="Kept as signature + docstring only; same sort order"
109
+ )
110
+ token_count: int = Field(description="Includes skeleton tokens; must be <= budget_tokens")
111
+ rendered_text: str = Field(description="The text that would be sent to the model")
112
+
113
+ # ---------- Gateway requests (Iteration 2) ----------
114
+ # A provider request (Anthropic / OpenAI / Gemini) is split into Segments by a provider
115
+ # adapter. Trimming and routing only ever see Segments, so they are written once for all
116
+ # providers. `path` locates the segment's text inside the original JSON body, so an adapter
117
+ # can write shortened text back without understanding the trimming.
118
+
119
+
120
+ class SegmentKind(str, Enum):
121
+ SYSTEM = "system" # system prompt / instructions (never trimmed)
122
+ TOOLS = "tools" # tool definitions (never trimmed)
123
+ USER_TEXT = "user_text"
124
+ ASSISTANT_TEXT = "assistant_text"
125
+ THINKING = "thinking" # model reasoning blocks (never trimmed)
126
+ TOOL_CALL = "tool_call" # assistant asks to run a tool (never trimmed)
127
+ TOOL_RESULT = "tool_result" # output of a tool: file contents, command output, ...
128
+ MEDIA = "media" # images, documents, unknown block types: never trimmed
129
+
130
+
131
+ class ToolResultKind(str, Enum):
132
+ FILE_READ = "file_read"
133
+ COMMAND_OUTPUT = "command_output"
134
+ SEARCH = "search"
135
+ OTHER = "other"
136
+
137
+
138
+ class Segment(BaseModel):
139
+ id: str = Field(description="Unique within one request, e.g. 'm3.b1' (message 3, block 1)")
140
+ kind: SegmentKind
141
+ role: str = Field(description="Provider role: system / user / assistant / tool / model")
142
+ message_index: int = Field(ge=-1, description="Index in the message list; -1 = outside it")
143
+ path: list[str | int] = Field(description="JSON path to this segment's text in the body")
144
+ text: str = ""
145
+ tool_name: str = ""
146
+ tool_call_id: str = ""
147
+ result_kind: ToolResultKind | None = None
148
+ token_count: int = Field(ge=0)
File without changes
@@ -0,0 +1,14 @@
1
+ """Run the gateway: python -m tokencut.gateway (listens on 127.0.0.1:8787 by default)."""
2
+
3
+ import os
4
+
5
+ import uvicorn
6
+
7
+ from tokencut.gateway.app import create_app
8
+
9
+ if __name__ == "__main__":
10
+ uvicorn.run(
11
+ create_app(),
12
+ host=os.environ.get("TOKENCUT_HOST", "127.0.0.1"), # local only unless asked otherwise
13
+ port=int(os.environ.get("TOKENCUT_PORT", "8787")),
14
+ )
@@ -0,0 +1 @@
1
+ """Provider adapters: turn a provider's request JSON into list[Segment] (and later back)."""
@@ -0,0 +1,119 @@
1
+ """Anthropic Messages API (POST /v1/messages): Claude Code and other Claude clients."""
2
+
3
+ from typing import Any
4
+
5
+ from tokencut.contracts import Segment, SegmentKind
6
+ from tokencut.gateway.adapters.common import SegmentBuilder, as_json
7
+
8
+ PROVIDER = "anthropic"
9
+
10
+
11
+ def model_of(body: dict, path: str) -> str:
12
+ return str(body.get("model", ""))
13
+
14
+
15
+ def parse(body: dict) -> list[Segment]:
16
+ b = SegmentBuilder()
17
+
18
+ system = body.get("system")
19
+ if isinstance(system, str):
20
+ b.add("system", SegmentKind.SYSTEM, "system", -1, ["system"], system)
21
+ elif isinstance(system, list):
22
+ for j, block in enumerate(system):
23
+ if isinstance(block, dict):
24
+ b.add(
25
+ f"system.{j}",
26
+ SegmentKind.SYSTEM,
27
+ "system",
28
+ -1,
29
+ ["system", j, "text"],
30
+ str(block.get("text", "")),
31
+ )
32
+
33
+ if body.get("tools"):
34
+ b.add("tools", SegmentKind.TOOLS, "system", -1, ["tools"], as_json(body["tools"]))
35
+
36
+ tool_names: dict[str, str] = {} # tool_use id -> tool name (results only carry the id)
37
+ for i, message in enumerate(body.get("messages", [])):
38
+ role = str(message.get("role", ""))
39
+ # Claude Code sends its environment / skills list as role "system" messages: agent
40
+ # context, not the user's words (as role "system" / "developer" in the OpenAI adapter).
41
+ text_kind = {"assistant": SegmentKind.ASSISTANT_TEXT,
42
+ "system": SegmentKind.SYSTEM}.get(role, SegmentKind.USER_TEXT)
43
+ content = message.get("content")
44
+ if isinstance(content, str):
45
+ b.add(f"m{i}", text_kind, role, i, ["messages", i, "content"], content)
46
+ continue
47
+ if not isinstance(content, list):
48
+ continue
49
+ for j, block in enumerate(content):
50
+ _parse_block(b, block, role, i, j, text_kind, tool_names)
51
+ return b.segments
52
+
53
+
54
+ def _parse_block(
55
+ b: SegmentBuilder,
56
+ block: Any,
57
+ role: str,
58
+ i: int,
59
+ j: int,
60
+ text_kind: SegmentKind,
61
+ tool_names: dict[str, str],
62
+ ) -> None:
63
+ if not isinstance(block, dict):
64
+ return
65
+ seg_id = f"m{i}.b{j}"
66
+ path: list[str | int] = ["messages", i, "content", j]
67
+ block_type = block.get("type")
68
+ if block_type == "text":
69
+ b.add(seg_id, text_kind, role, i, path + ["text"], str(block.get("text", "")))
70
+ elif block_type == "thinking":
71
+ b.add(
72
+ seg_id,
73
+ SegmentKind.THINKING,
74
+ role,
75
+ i,
76
+ path + ["thinking"],
77
+ str(block.get("thinking", "")),
78
+ )
79
+ elif block_type == "redacted_thinking":
80
+ b.add(seg_id, SegmentKind.THINKING, role, i, path, "")
81
+ elif block_type == "tool_use":
82
+ call_id = str(block.get("id", ""))
83
+ name = str(block.get("name", ""))
84
+ tool_names[call_id] = name
85
+ b.add(
86
+ seg_id,
87
+ SegmentKind.TOOL_CALL,
88
+ role,
89
+ i,
90
+ path + ["input"],
91
+ as_json(block.get("input", {})),
92
+ name,
93
+ call_id,
94
+ )
95
+ elif block_type == "tool_result":
96
+ call_id = str(block.get("tool_use_id", ""))
97
+ name = tool_names.get(call_id, "")
98
+ inner = block.get("content", "")
99
+ if isinstance(inner, str):
100
+ b.add_tool_result(seg_id, role, i, path + ["content"], inner, name, call_id)
101
+ elif isinstance(inner, list):
102
+ for k, sub in enumerate(inner):
103
+ sub_id = f"{seg_id}.c{k}"
104
+ if isinstance(sub, dict) and sub.get("type") == "text":
105
+ b.add_tool_result(
106
+ sub_id,
107
+ role,
108
+ i,
109
+ path + ["content", k, "text"],
110
+ str(sub.get("text", "")),
111
+ name,
112
+ call_id,
113
+ )
114
+ else:
115
+ b.add(sub_id, SegmentKind.MEDIA, role, i, path + ["content", k], "")
116
+ else:
117
+ # image, document, server tool blocks, future block types: counted, never trimmed
118
+ text = "" if block_type in ("image", "document") else as_json(block)
119
+ b.add(seg_id, SegmentKind.MEDIA, role, i, path, text)
@@ -0,0 +1,95 @@
1
+ """Helpers shared by the provider adapters."""
2
+
3
+ import json
4
+ from typing import Any
5
+
6
+ from tokencut.contracts import Segment, SegmentKind, ToolResultKind
7
+ from tokencut.gateway.classify import classify_tool_result
8
+ from tokencut.tokens import count_tokens
9
+
10
+ JsonPath = list[str | int]
11
+
12
+
13
+ def as_json(value: Any) -> str:
14
+ """Stable text form of a JSON value, for token counting."""
15
+ return json.dumps(value, ensure_ascii=False, sort_keys=True)
16
+
17
+
18
+ def get(obj: dict, camel: str, snake: str, default: Any = None) -> Any:
19
+ """Gemini's REST API accepts both camelCase and snake_case field names."""
20
+ if camel in obj:
21
+ return obj[camel]
22
+ return obj.get(snake, default)
23
+
24
+
25
+ def key_name(obj: dict, camel: str, snake: str) -> str:
26
+ return camel if camel in obj else snake
27
+
28
+
29
+ class SegmentBuilder:
30
+ """Collects segments with unique ids and token counts."""
31
+
32
+ def __init__(self) -> None:
33
+ self.segments: list[Segment] = []
34
+
35
+ def add(
36
+ self,
37
+ seg_id: str,
38
+ kind: SegmentKind,
39
+ role: str,
40
+ message_index: int,
41
+ path: JsonPath,
42
+ text: str,
43
+ tool_name: str = "",
44
+ tool_call_id: str = "",
45
+ result_kind: ToolResultKind | None = None,
46
+ ) -> None:
47
+ self.segments.append(
48
+ Segment(
49
+ id=seg_id,
50
+ kind=kind,
51
+ role=role,
52
+ message_index=message_index,
53
+ path=list(path),
54
+ text=text,
55
+ tool_name=tool_name,
56
+ tool_call_id=tool_call_id,
57
+ result_kind=result_kind,
58
+ token_count=count_tokens(text),
59
+ )
60
+ )
61
+
62
+ def add_tool_result(
63
+ self,
64
+ seg_id: str,
65
+ role: str,
66
+ message_index: int,
67
+ path: JsonPath,
68
+ text: str,
69
+ tool_name: str,
70
+ tool_call_id: str,
71
+ ) -> None:
72
+ self.add(
73
+ seg_id,
74
+ SegmentKind.TOOL_RESULT,
75
+ role,
76
+ message_index,
77
+ path,
78
+ text,
79
+ tool_name,
80
+ tool_call_id,
81
+ classify_tool_result(tool_name, text),
82
+ )
83
+
84
+
85
+ def lookup(body: Any, path: JsonPath) -> Any:
86
+ value = body
87
+ for key in path:
88
+ value = value[key]
89
+ return value
90
+
91
+
92
+ def set_text(body: Any, path: JsonPath, text: str) -> None:
93
+ """Replace the string at `path` (the location a Segment points to)."""
94
+ parent = lookup(body, path[:-1])
95
+ parent[path[-1]] = text
@@ -0,0 +1,107 @@
1
+ """Google Gemini API (POST /v1beta/models/{model}:generateContent / :streamGenerateContent).
2
+
3
+ Gemini CLI and google-genai clients. The REST API accepts camelCase and snake_case keys.
4
+ """
5
+
6
+ import re
7
+
8
+ from tokencut.contracts import Segment, SegmentKind
9
+ from tokencut.gateway.adapters.common import SegmentBuilder, as_json, get, key_name
10
+
11
+ PROVIDER = "gemini"
12
+ _MODEL_IN_PATH = re.compile(r"models/([^/:]+):")
13
+ # Common single-string fields in functionResponse.response (Gemini CLI uses "output").
14
+ _RESULT_TEXT_FIELDS = ("output", "content", "result", "text")
15
+
16
+
17
+ def model_of(body: dict, path: str) -> str:
18
+ match = _MODEL_IN_PATH.search(path)
19
+ return match.group(1) if match else ""
20
+
21
+
22
+ def parse(body: dict) -> list[Segment]:
23
+ b = SegmentBuilder()
24
+ system_key = key_name(body, "systemInstruction", "system_instruction")
25
+ system = body.get(system_key)
26
+ if isinstance(system, dict):
27
+ for j, part in enumerate(system.get("parts", [])):
28
+ if isinstance(part, dict) and "text" in part:
29
+ b.add(
30
+ f"system.{j}",
31
+ SegmentKind.SYSTEM,
32
+ "system",
33
+ -1,
34
+ [system_key, "parts", j, "text"],
35
+ str(part["text"]),
36
+ )
37
+ if body.get("tools"):
38
+ b.add("tools", SegmentKind.TOOLS, "system", -1, ["tools"], as_json(body["tools"]))
39
+
40
+ for i, content in enumerate(body.get("contents", [])):
41
+ role = str(content.get("role", "user"))
42
+ text_kind = SegmentKind.ASSISTANT_TEXT if role == "model" else SegmentKind.USER_TEXT
43
+ for j, part in enumerate(content.get("parts", [])):
44
+ if not isinstance(part, dict):
45
+ continue
46
+ seg_id = f"m{i}.p{j}"
47
+ path: list[str | int] = ["contents", i, "parts", j]
48
+ call = get(part, "functionCall", "function_call")
49
+ response = get(part, "functionResponse", "function_response")
50
+ if "text" in part:
51
+ kind = SegmentKind.THINKING if part.get("thought") else text_kind
52
+ b.add(seg_id, kind, role, i, path + ["text"], str(part["text"]))
53
+ elif isinstance(call, dict):
54
+ b.add(
55
+ seg_id,
56
+ SegmentKind.TOOL_CALL,
57
+ role,
58
+ i,
59
+ path + [key_name(part, "functionCall", "function_call"), "args"],
60
+ as_json(call.get("args", {})),
61
+ str(call.get("name", "")),
62
+ str(call.get("id", "")),
63
+ )
64
+ elif isinstance(response, dict):
65
+ fr_key = key_name(part, "functionResponse", "function_response")
66
+ name = str(response.get("name", ""))
67
+ call_id = str(response.get("id", ""))
68
+ payload = response.get("response", {})
69
+ field = _text_field(payload)
70
+ if field is not None:
71
+ b.add_tool_result(
72
+ seg_id,
73
+ role,
74
+ i,
75
+ path + [fr_key, "response", field],
76
+ payload[field],
77
+ name,
78
+ call_id,
79
+ )
80
+ else:
81
+ b.add_tool_result(
82
+ seg_id,
83
+ role,
84
+ i,
85
+ path + [fr_key, "response"],
86
+ as_json(payload),
87
+ name,
88
+ call_id,
89
+ )
90
+ # Gemini CLI nests a tool's images / documents here on multimodal models
91
+ for m, nested in enumerate(response.get("parts") or []):
92
+ if isinstance(nested, dict):
93
+ b.add(f"{seg_id}.n{m}", SegmentKind.MEDIA, role, i,
94
+ path + [fr_key, "parts", m], "")
95
+ else: # inlineData, fileData, executableCode, ...
96
+ b.add(seg_id, SegmentKind.MEDIA, role, i, path, "")
97
+ return b.segments
98
+
99
+
100
+ def _text_field(payload: object) -> str | None:
101
+ """The one string field holding the tool output, if the response has that shape."""
102
+ if not isinstance(payload, dict):
103
+ return None
104
+ for field in _RESULT_TEXT_FIELDS:
105
+ if isinstance(payload.get(field), str) and len(payload) <= 2:
106
+ return field
107
+ return None