kvsig 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
kvsig/__init__.py ADDED
@@ -0,0 +1,128 @@
1
+ """kvsig: keyed, content-derived ids for LLM chat conversations.
2
+
3
+ Reference implementation of the kvsig specification (``kvsig/1``, draft 0.1).
4
+ Pure functions, no third-party dependencies. Hostile input never raises: it
5
+ yields ``None``.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+ from collections.abc import Callable
12
+ from typing import Any, TypeVar
13
+
14
+ from kvsig.cbor import encode
15
+ from kvsig.dialects import SUPPORTED, Dialect
16
+ from kvsig.dialects import anthropic_messages as _anthropic_messages
17
+ from kvsig.dialects import openai_chat as _openai_chat
18
+ from kvsig.dialects import openai_responses as _openai_responses
19
+ from kvsig.ids import SPEC_LABEL, IdCache, Ids, ids, ids_all_epochs, message_id, scope_from
20
+ from kvsig.keys import TEST_KEY, InMemoryKeyProvider, Key, KeyProvider, KeyRing
21
+ from kvsig.loose import LOOSE_V1, LOOSE_V2, LooseProfile, loose
22
+ from kvsig.model import DEFAULT_MAX_BYTES, Budget, Canon, Message, NoIds, finish
23
+
24
+ __version__ = "0.1.0"
25
+
26
+ __all__ = [
27
+ "DEFAULT_MAX_BYTES",
28
+ "LOOSE_V1",
29
+ "LOOSE_V2",
30
+ "SUPPORTED",
31
+ "SPEC_LABEL",
32
+ "TEST_KEY",
33
+ "__version__",
34
+ "Canon",
35
+ "Dialect",
36
+ "IdCache",
37
+ "Ids",
38
+ "InMemoryKeyProvider",
39
+ "Key",
40
+ "KeyProvider",
41
+ "KeyRing",
42
+ "LooseProfile",
43
+ "Message",
44
+ "canonicalize",
45
+ "canonicalize_reply",
46
+ "conversation_id",
47
+ "encode",
48
+ "ids",
49
+ "ids_all_epochs",
50
+ "loose",
51
+ "message_id",
52
+ "scope_from",
53
+ ]
54
+
55
+ log = logging.getLogger("kvsig")
56
+
57
+ T = TypeVar("T")
58
+
59
+
60
+ _ADAPTERS: dict[str, Any] = {
61
+ "openai.chat": _openai_chat,
62
+ "anthropic.messages": _anthropic_messages,
63
+ "openai.responses": _openai_responses,
64
+ }
65
+
66
+
67
+ assert set(_ADAPTERS) == set(SUPPORTED), "dialects.SUPPORTED and _ADAPTERS disagree"
68
+
69
+
70
+ def _adapter(dialect: str) -> Any:
71
+ a = _ADAPTERS.get(dialect)
72
+ if a is None:
73
+ raise NotImplementedError(f"dialect {dialect!r} is not implemented in this release")
74
+ return a
75
+
76
+
77
+ def _fail_open(what: str, fn: Callable[[], T | None]) -> T | None:
78
+ """Run ``fn``; the defined "no ids" outcomes return None quietly. Anything
79
+ else is a bug in this library: it still returns None (an id must never
80
+ fail a request) but is logged, so a broken deployment is visible."""
81
+ try:
82
+ return fn()
83
+ except (NoIds, RecursionError):
84
+ return None
85
+ except Exception: # noqa: BLE001 -- identity must never fail a request
86
+ log.exception("kvsig: unexpected error in %s; returning no ids", what)
87
+ return None
88
+
89
+
90
+ def canonicalize(
91
+ body: Any, dialect: Dialect = "openai.chat", *, max_bytes: int = DEFAULT_MAX_BYTES
92
+ ) -> Canon | None:
93
+ """The canonical form of a request body, or None when the request gets no
94
+ ids: not a chat body, over ``max_bytes`` of canonical CBOR, or nested too
95
+ deep. Raises only for an unknown ``dialect``."""
96
+ adapter = _adapter(dialect)
97
+
98
+ def run() -> Canon | None:
99
+ if not isinstance(body, dict):
100
+ return None
101
+ c = adapter.canonicalize(body, Budget(max_bytes))
102
+ return None if c is None else finish(c, max_bytes)
103
+
104
+ return _fail_open("canonicalize", run)
105
+
106
+
107
+ def canonicalize_reply(
108
+ message: Any, dialect: Dialect = "openai.chat", *, max_bytes: int = DEFAULT_MAX_BYTES
109
+ ) -> Message | None:
110
+ """A response's assistant message, canonicalised alone, so that its
111
+ ``message_id`` equals the id of the same message in the next request.
112
+
113
+ ``message`` is the assistant message (``openai.chat``: ``choices[i].message``),
114
+ the whole response (``anthropic.messages``), or the whole response with its
115
+ ``output`` items (``openai.responses``)."""
116
+ adapter = _adapter(dialect)
117
+
118
+ def run() -> Message | None:
119
+ if not isinstance(message, dict):
120
+ return None
121
+ m = adapter.reply(message, Budget(max_bytes))
122
+ return m if len(encode(m)) <= max_bytes else None
123
+
124
+ return _fail_open("canonicalize_reply", run)
125
+
126
+
127
+ def conversation_id(i: Ids) -> str | None:
128
+ return i.conv
kvsig/_text.py ADDED
@@ -0,0 +1,33 @@
1
+ """Unicode scalar handling shared by every encoder (decision 0001, rule 4)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+
7
+ # A well-formed surrogate pair, or any lone surrogate.
8
+ _SURROGATES = re.compile("[\ud800-\udbff][\udc00-\udfff]|[\ud800-\udfff]")
9
+
10
+
11
+ def _fix(m: re.Match[str]) -> str:
12
+ g = m.group()
13
+ if len(g) == 2:
14
+ return chr(0x10000 + ((ord(g[0]) - 0xD800) << 10) + (ord(g[1]) - 0xDC00))
15
+ return "\ufffd"
16
+
17
+
18
+ def scalar(s: str) -> str:
19
+ """``s`` as Unicode scalar values: a surrogate pair becomes the code point it
20
+ stands for, a lone surrogate becomes U+FFFD. No normalisation."""
21
+ try:
22
+ s.encode("utf-8")
23
+ except UnicodeEncodeError:
24
+ return _SURROGATES.sub(_fix, s)
25
+ return s
26
+
27
+
28
+ def utf8(s: str) -> bytes:
29
+ """UTF-8 bytes of ``scalar(s)``."""
30
+ try:
31
+ return s.encode("utf-8")
32
+ except UnicodeEncodeError:
33
+ return _SURROGATES.sub(_fix, s).encode("utf-8")
kvsig/cbor.py ADDED
@@ -0,0 +1,88 @@
1
+ """The restricted deterministic CBOR profile of decision 0001.
2
+
3
+ Allowed: integers, byte strings, text strings, arrays, maps with text keys,
4
+ ``True`` and ``False``. Everything is in preferred (shortest) serialisation and
5
+ map keys are sorted bytewise by their encoding (RFC 8949 section 4.2.1).
6
+ Anything else (floats, ``None``, other key types) raises ``TypeError``.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from typing import Any
12
+
13
+ from kvsig._text import utf8
14
+
15
+ _MAX_UINT = (1 << 64) - 1
16
+
17
+
18
+ def _head(major: int, n: int, out: bytearray) -> None:
19
+ mt = major << 5
20
+ if n < 24:
21
+ out.append(mt | n)
22
+ elif n < 0x100:
23
+ out.append(mt | 24)
24
+ out.append(n)
25
+ elif n < 0x10000:
26
+ out.append(mt | 25)
27
+ out += n.to_bytes(2, "big")
28
+ elif n < 0x100000000:
29
+ out.append(mt | 26)
30
+ out += n.to_bytes(4, "big")
31
+ elif n <= _MAX_UINT:
32
+ out.append(mt | 27)
33
+ out += n.to_bytes(8, "big")
34
+ else:
35
+ raise ValueError("integer out of CBOR range")
36
+
37
+
38
+ def _text(s: str, out: bytearray) -> None:
39
+ b = utf8(s)
40
+ _head(3, len(b), out)
41
+ out += b
42
+
43
+
44
+ def _enc(o: Any, out: bytearray) -> None:
45
+ # bool before int: bool is a subclass of int.
46
+ if o is True:
47
+ out.append(0xF5)
48
+ elif o is False:
49
+ out.append(0xF4)
50
+ elif isinstance(o, str):
51
+ _text(o, out)
52
+ elif isinstance(o, int):
53
+ if o >= 0:
54
+ _head(0, o, out)
55
+ else:
56
+ _head(1, -1 - o, out)
57
+ elif isinstance(o, (bytes, bytearray)):
58
+ _head(2, len(o), out)
59
+ out += o
60
+ elif isinstance(o, (list, tuple)):
61
+ _head(4, len(o), out)
62
+ for v in o:
63
+ _enc(v, out)
64
+ elif isinstance(o, dict):
65
+ items: list[tuple[bytes, Any]] = []
66
+ for k, v in o.items():
67
+ if not isinstance(k, str):
68
+ raise TypeError("CBOR map keys must be text")
69
+ kb = bytearray()
70
+ _text(k, kb)
71
+ items.append((bytes(kb), v))
72
+ items.sort(key=lambda kv: kv[0])
73
+ for i in range(1, len(items)):
74
+ if items[i][0] == items[i - 1][0]:
75
+ raise ValueError("duplicate CBOR map key after normalisation")
76
+ _head(5, len(items), out)
77
+ for kb, v in items:
78
+ out += kb
79
+ _enc(v, out)
80
+ else:
81
+ raise TypeError(f"type {type(o).__name__} is not in the kvsig CBOR profile")
82
+
83
+
84
+ def encode(obj: Any) -> bytes:
85
+ """The canonical bytes of ``obj``."""
86
+ out = bytearray()
87
+ _enc(obj, out)
88
+ return bytes(out)
@@ -0,0 +1,10 @@
1
+ """Dialect adapters: one module per API request format."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Literal
6
+
7
+ Dialect = Literal["openai.chat", "openai.responses", "anthropic.messages", "openai.completions"]
8
+
9
+ #: Dialects this release implements.
10
+ SUPPORTED: tuple[str, ...] = ("openai.chat", "openai.responses", "anthropic.messages")
@@ -0,0 +1,225 @@
1
+ """Anthropic Messages request and response bodies to the canonical form.
2
+
3
+ The mapping is chosen so that a conversation sent through OpenAI Chat
4
+ Completions and through Anthropic Messages gives the same canonical form:
5
+
6
+ - ``system`` (a string or text blocks) is the head's one system message.
7
+ - A user turn's ``tool_result`` blocks each become their own ``tool`` message,
8
+ in place; the other blocks of the turn stay together as user messages. That
9
+ is how Chat Completions expresses the same turn.
10
+ - ``thinking`` blocks are reasoning parts, at their position; the signature
11
+ is not part of the conversation. ``redacted_thinking`` is opaque.
12
+ - ``cache_control`` and ``citations`` are ignored everywhere.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import binascii
18
+ from typing import Any
19
+
20
+ from kvsig.model import (
21
+ HEAD_ROLES,
22
+ Budget,
23
+ Canon,
24
+ Message,
25
+ Part,
26
+ b64decode,
27
+ json_text,
28
+ media_bytes,
29
+ media_ref,
30
+ opaque,
31
+ text_part,
32
+ without_cache_control,
33
+ )
34
+
35
+
36
+ def _source(block: dict[str, Any], budget: Budget) -> Part:
37
+ """An ``image`` or ``document`` block with a plain source."""
38
+ src = block.get("source")
39
+ if not isinstance(src, dict):
40
+ return opaque(block, budget)
41
+ kind = src.get("type")
42
+ mime = src.get("media_type")
43
+ mime = mime if isinstance(mime, str) else ""
44
+ if kind == "base64":
45
+ data = src.get("data")
46
+ try:
47
+ return media_bytes(b64decode(data), mime)
48
+ except (binascii.Error, ValueError):
49
+ return opaque(block, budget)
50
+ if kind == "text" and isinstance(src.get("data"), str):
51
+ return media_bytes(src["data"].encode("utf-8", "surrogatepass"), mime)
52
+ if kind == "url" and isinstance(src.get("url"), str):
53
+ return media_ref(src["url"], "url")
54
+ if kind == "file" and isinstance(src.get("file_id"), str):
55
+ return media_ref(src["file_id"], "file_id")
56
+ return opaque(block, budget)
57
+
58
+
59
+ def _content_block(b: Any, budget: Budget) -> Part | None:
60
+ """A user or tool-result block other than ``tool_result``. None means an
61
+ empty text, which is dropped."""
62
+ if isinstance(b, str):
63
+ return text_part(b, budget)
64
+ if not isinstance(b, dict):
65
+ return opaque(b, budget, "block")
66
+ t = b.get("type")
67
+ if t == "text":
68
+ v = b.get("text")
69
+ return text_part(v if isinstance(v, str) else "", budget)
70
+ if t == "image":
71
+ return _source(b, budget)
72
+ if t == "document" and not any(k in b for k in ("title", "context")):
73
+ # A document with a title or context carries text the model reads
74
+ # beside the file: keep all of it, as opaque.
75
+ return _source(b, budget)
76
+ return opaque(b, budget)
77
+
78
+
79
+ def _blocks(content: Any, budget: Budget) -> list[Part]:
80
+ if content is None:
81
+ return []
82
+ if isinstance(content, str):
83
+ p = text_part(content, budget)
84
+ return [p] if p else []
85
+ if not isinstance(content, list):
86
+ return [opaque(content, budget, "content")]
87
+ out: list[Part] = []
88
+ for b in content:
89
+ p = _content_block(b, budget)
90
+ if p is not None:
91
+ out.append(p)
92
+ return out
93
+
94
+
95
+ def _user_turn(role: str, content: Any, seen: dict[str, int], budget: Budget) -> list[Message]:
96
+ """One user turn, split so each ``tool_result`` is its own message."""
97
+ if not isinstance(content, list):
98
+ return [{"role": role, "parts": _blocks(content, budget)}]
99
+ out: list[Message] = []
100
+ pending: list[Part] = []
101
+ pending_open = False # a user message has started (even if all its texts were empty)
102
+
103
+ def flush() -> None:
104
+ nonlocal pending, pending_open
105
+ if pending_open:
106
+ out.append({"role": role, "parts": pending})
107
+ pending, pending_open = [], False
108
+
109
+ for b in content:
110
+ if isinstance(b, dict) and b.get("type") == "tool_result":
111
+ flush()
112
+ tid = b.get("tool_use_id")
113
+ ref = seen.get(tid, -1) if isinstance(tid, str) else -1
114
+ result: Part = {
115
+ "t": "tool_result",
116
+ "ref": ref,
117
+ "parts": _blocks(b.get("content"), budget),
118
+ }
119
+ if b.get("is_error") is True:
120
+ result["error"] = True
121
+ out.append({"role": "tool", "parts": [result]})
122
+ else:
123
+ pending_open = True
124
+ p = _content_block(b, budget)
125
+ if p is not None:
126
+ pending.append(p)
127
+ flush()
128
+ if not out:
129
+ out.append({"role": role, "parts": []})
130
+ return out
131
+
132
+
133
+ def _assistant_turn(role: str, content: Any, seen: dict[str, int], budget: Budget) -> Message:
134
+ if not isinstance(content, list):
135
+ return {"role": role, "parts": _blocks(content, budget)}
136
+ parts: list[Part] = []
137
+ index = 0
138
+ for b in content:
139
+ if not isinstance(b, dict):
140
+ parts.append(opaque(b, budget, "block"))
141
+ continue
142
+ t = b.get("type")
143
+ if t == "thinking":
144
+ v = b.get("thinking")
145
+ if isinstance(v, str) and v:
146
+ budget.charge(len(v))
147
+ parts.append({"t": "reasoning", "v": v})
148
+ elif t == "text":
149
+ p = text_part(b.get("text") if isinstance(b.get("text"), str) else "", budget)
150
+ if p is not None:
151
+ parts.append(p)
152
+ elif t == "tool_use":
153
+ name = b.get("name")
154
+ name = name if isinstance(name, str) else ""
155
+ budget.charge(len(name))
156
+ parts.append(
157
+ {
158
+ "t": "tool_call",
159
+ "name": name,
160
+ "args": json_text(b.get("input"), budget),
161
+ "ref": index,
162
+ }
163
+ )
164
+ tid = b.get("id")
165
+ if isinstance(tid, str):
166
+ seen[tid] = index
167
+ index += 1
168
+ else:
169
+ # redacted_thinking, server_tool_use, *_tool_result, and anything newer
170
+ parts.append(opaque(b, budget))
171
+ return {"role": role, "parts": parts}
172
+
173
+
174
+ def message(m: Any, seen: dict[str, int], budget: Budget) -> list[Message]:
175
+ if not isinstance(m, dict):
176
+ return [{"role": "", "parts": [opaque(m, budget, "message")]}]
177
+ role = m.get("role")
178
+ role = role if isinstance(role, str) else ""
179
+ budget.charge(len(role))
180
+ if role == "assistant":
181
+ return [_assistant_turn(role, m.get("content"), seen, budget)]
182
+ return _user_turn(role, m.get("content"), seen, budget)
183
+
184
+
185
+ def _system(system: Any, budget: Budget) -> list[Message]:
186
+ if system is None or system == "" or system == []:
187
+ return []
188
+ return [{"role": "system", "parts": _blocks(system, budget)}]
189
+
190
+
191
+ def _tool(t: Any, budget: Budget) -> dict[str, Any]:
192
+ if isinstance(t, dict) and t.get("type") in (None, "custom") and isinstance(t.get("name"), str):
193
+ name, desc = t["name"], t.get("description")
194
+ desc = desc if isinstance(desc, str) else ""
195
+ budget.charge(len(name) + len(desc))
196
+ return {"name": name, "desc": desc, "schema": json_text(t.get("input_schema"), budget)}
197
+ t_type = t.get("type") if isinstance(t, dict) else None
198
+ return {
199
+ "type": t_type if isinstance(t_type, str) else "",
200
+ "def": json_text(without_cache_control(t), budget),
201
+ }
202
+
203
+
204
+ def canonicalize(body: dict[str, Any], budget: Budget) -> Canon | None:
205
+ msgs_in = body.get("messages")
206
+ if not isinstance(msgs_in, list):
207
+ return None
208
+ tools_in = body.get("tools")
209
+ tools = [_tool(t, budget) for t in tools_in] if isinstance(tools_in, list) else []
210
+ head = _system(body.get("system"), budget)
211
+ body_msgs: list[Message] = []
212
+ seen: dict[str, int] = {}
213
+ for m in msgs_in:
214
+ for c in message(m, seen, budget):
215
+ if not body_msgs and c["role"] in HEAD_ROLES:
216
+ head.append(c)
217
+ else:
218
+ body_msgs.append(c)
219
+ return Canon(head=head, tools=tools, messages=body_msgs)
220
+
221
+
222
+ def reply(m: dict[str, Any], budget: Budget) -> Message:
223
+ """A Messages response (or its assistant message), canonicalised alone."""
224
+ role = m.get("role") if isinstance(m.get("role"), str) else "assistant"
225
+ return _assistant_turn(role, m.get("content"), {}, budget)