gcf-python 2.2.2__py3-none-any.whl → 2.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gcf/__init__.py +33 -1
- gcf/generic_delta.py +487 -0
- {gcf_python-2.2.2.dist-info → gcf_python-2.3.0.dist-info}/METADATA +51 -7
- {gcf_python-2.2.2.dist-info → gcf_python-2.3.0.dist-info}/RECORD +7 -6
- {gcf_python-2.2.2.dist-info → gcf_python-2.3.0.dist-info}/WHEEL +0 -0
- {gcf_python-2.2.2.dist-info → gcf_python-2.3.0.dist-info}/entry_points.txt +0 -0
- {gcf_python-2.2.2.dist-info → gcf_python-2.3.0.dist-info}/licenses/LICENSE +0 -0
gcf/__init__.py
CHANGED
|
@@ -39,6 +39,23 @@ from .decode import DecodeError, decode
|
|
|
39
39
|
from .delta import encode_delta
|
|
40
40
|
from .encode import encode
|
|
41
41
|
from .generic import encode_generic, GenericOptions
|
|
42
|
+
from .generic_delta import (
|
|
43
|
+
GenericSet,
|
|
44
|
+
GenericDeltaPayload,
|
|
45
|
+
generic_pack_root,
|
|
46
|
+
diff_generic_sets,
|
|
47
|
+
encode_generic_full,
|
|
48
|
+
encode_generic_delta,
|
|
49
|
+
decode_generic_full,
|
|
50
|
+
decode_generic_delta,
|
|
51
|
+
verify_generic_delta,
|
|
52
|
+
GenericDeltaSession,
|
|
53
|
+
ReanchorPolicy,
|
|
54
|
+
ReanchorMode,
|
|
55
|
+
fixed_n,
|
|
56
|
+
size_guard,
|
|
57
|
+
DEFAULT_REANCHOR_N,
|
|
58
|
+
)
|
|
42
59
|
from .session import Session, encode_with_session
|
|
43
60
|
from .decode_generic import decode_generic
|
|
44
61
|
from .stream import StreamEncoder
|
|
@@ -64,6 +81,21 @@ __all__ = [
|
|
|
64
81
|
"encode_generic",
|
|
65
82
|
"GenericOptions",
|
|
66
83
|
"encode_with_session",
|
|
84
|
+
"GenericSet",
|
|
85
|
+
"GenericDeltaPayload",
|
|
86
|
+
"generic_pack_root",
|
|
87
|
+
"diff_generic_sets",
|
|
88
|
+
"encode_generic_full",
|
|
89
|
+
"encode_generic_delta",
|
|
90
|
+
"decode_generic_full",
|
|
91
|
+
"decode_generic_delta",
|
|
92
|
+
"verify_generic_delta",
|
|
93
|
+
"GenericDeltaSession",
|
|
94
|
+
"ReanchorPolicy",
|
|
95
|
+
"ReanchorMode",
|
|
96
|
+
"fixed_n",
|
|
97
|
+
"size_guard",
|
|
98
|
+
"DEFAULT_REANCHOR_N",
|
|
67
99
|
]
|
|
68
100
|
|
|
69
|
-
__version__ = "2.
|
|
101
|
+
__version__ = "2.3.0"
|
gcf/generic_delta.py
ADDED
|
@@ -0,0 +1,487 @@
|
|
|
1
|
+
"""GCF generic-profile delta encoding (SPEC Section 10a).
|
|
2
|
+
|
|
3
|
+
Producer + consumer. Mirrors the gcf-go reference implementation; the shared
|
|
4
|
+
conformance fixtures (generic-pack-root/, generic-delta/) hold both to identical
|
|
5
|
+
bytes and hashes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from enum import Enum
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from .scalar import (
|
|
16
|
+
format_key,
|
|
17
|
+
format_number,
|
|
18
|
+
format_scalar,
|
|
19
|
+
parse_scalar,
|
|
20
|
+
quote_string,
|
|
21
|
+
split_respecting_quotes,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class GenericSet:
|
|
27
|
+
"""A keyed record set: the unit generic-profile delta operates on (Section 10a).
|
|
28
|
+
|
|
29
|
+
Rows are order-agnostic (set semantics); ``fields`` carries the declared column
|
|
30
|
+
order for the wire form; ``key`` names the identity column (the ``@id`` / ``key=``);
|
|
31
|
+
``name`` is the tabular section name for a full payload.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
key: str
|
|
35
|
+
fields: list[str]
|
|
36
|
+
rows: list[dict[str, Any]]
|
|
37
|
+
name: str = "rows"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class GenericDeltaPayload:
|
|
42
|
+
key: str
|
|
43
|
+
fields: list[str]
|
|
44
|
+
base_root: str = ""
|
|
45
|
+
new_root: str = ""
|
|
46
|
+
added: list[dict[str, Any]] = field(default_factory=list)
|
|
47
|
+
changed: list[dict[str, Any]] = field(default_factory=list)
|
|
48
|
+
removed: list[Any] = field(default_factory=list)
|
|
49
|
+
tool: str = ""
|
|
50
|
+
delta_tokens: int = 0
|
|
51
|
+
full_tokens: int = 0
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _canonical_cell(v: Any) -> str:
|
|
55
|
+
"""Canonicalize one value for the pack-root record (Section 10a.3).
|
|
56
|
+
|
|
57
|
+
Decoupled from the wire cell encoder: collision-free and record-safe, not
|
|
58
|
+
round-trippable. Typed literals stay bare (null is ``-``, booleans true/false,
|
|
59
|
+
numbers canonical); strings are ALWAYS quoted so they cannot collide with a
|
|
60
|
+
typed literal and any tab/newline inside is escaped.
|
|
61
|
+
"""
|
|
62
|
+
if v is None:
|
|
63
|
+
return "-"
|
|
64
|
+
if isinstance(v, bool): # must precede int: bool is a subclass of int
|
|
65
|
+
return "true" if v else "false"
|
|
66
|
+
if isinstance(v, (int, float)):
|
|
67
|
+
return format_number(float(v))
|
|
68
|
+
if isinstance(v, str):
|
|
69
|
+
return quote_string(v)
|
|
70
|
+
return quote_string(str(v))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def generic_pack_root(s: GenericSet) -> str:
|
|
74
|
+
"""Canonical pack root for a keyed set (gcf-pack-root-v1, generic profile, 10a.3).
|
|
75
|
+
|
|
76
|
+
Records and fields are sorted by unsigned UTF-8 byte order to match every SDK.
|
|
77
|
+
"""
|
|
78
|
+
sorted_fields = sorted(s.fields, key=lambda x: x.encode("utf-8"))
|
|
79
|
+
records: list[str] = []
|
|
80
|
+
for row in s.rows:
|
|
81
|
+
parts = ["R"]
|
|
82
|
+
for f in sorted_fields:
|
|
83
|
+
parts.append(f)
|
|
84
|
+
parts.append(_canonical_cell(row.get(f)))
|
|
85
|
+
records.append("\t".join(parts) + "\n")
|
|
86
|
+
records.sort(key=lambda r: r.encode("utf-8"))
|
|
87
|
+
digest = hashlib.sha256("".join(records).encode("utf-8")).hexdigest()
|
|
88
|
+
return "sha256:" + digest
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _index_by_key(s: GenericSet) -> dict[str, dict[str, Any]]:
|
|
92
|
+
m: dict[str, dict[str, Any]] = {}
|
|
93
|
+
for row in s.rows:
|
|
94
|
+
ident = _canonical_cell(row.get(s.key))
|
|
95
|
+
if ident in m:
|
|
96
|
+
raise ValueError(f"delta_invalid: duplicate identity {ident} for key {s.key!r}")
|
|
97
|
+
m[ident] = row
|
|
98
|
+
return m
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _rows_equal(a: dict[str, Any], b: dict[str, Any], fields: list[str]) -> bool:
|
|
102
|
+
return all(_canonical_cell(a.get(f)) == _canonical_cell(b.get(f)) for f in fields)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def diff_generic_sets(base: GenericSet, nxt: GenericSet) -> GenericDeltaPayload:
|
|
106
|
+
"""Compute the delta from ``base`` to ``nxt`` (the blessed producer path).
|
|
107
|
+
|
|
108
|
+
Centralizes the keyed-diff invariants: identity uniqueness, added-not-in-base,
|
|
109
|
+
changed-must-exist, whole-row replacement, unchanged rows omitted. Output is
|
|
110
|
+
sorted by identity for reproducibility (10a.6). Schema change or a missing key
|
|
111
|
+
raises (caller must send full, 10a.7).
|
|
112
|
+
"""
|
|
113
|
+
if not nxt.key:
|
|
114
|
+
raise ValueError("delta_invalid: no identity key")
|
|
115
|
+
if nxt.key != base.key or list(base.fields) != list(nxt.fields):
|
|
116
|
+
raise ValueError("delta_invalid: schema change (send full)")
|
|
117
|
+
|
|
118
|
+
base_by_id = _index_by_key(base)
|
|
119
|
+
next_by_id = _index_by_key(nxt)
|
|
120
|
+
|
|
121
|
+
d = GenericDeltaPayload(
|
|
122
|
+
key=nxt.key,
|
|
123
|
+
fields=list(nxt.fields),
|
|
124
|
+
base_root=generic_pack_root(base),
|
|
125
|
+
new_root=generic_pack_root(nxt),
|
|
126
|
+
)
|
|
127
|
+
for ident, row in next_by_id.items():
|
|
128
|
+
brow = base_by_id.get(ident)
|
|
129
|
+
if brow is None:
|
|
130
|
+
d.added.append(row)
|
|
131
|
+
elif not _rows_equal(brow, row, nxt.fields):
|
|
132
|
+
d.changed.append(row)
|
|
133
|
+
# equal rows are omitted (silence = "keep it", 10a.5)
|
|
134
|
+
for ident, brow in base_by_id.items():
|
|
135
|
+
if ident not in next_by_id:
|
|
136
|
+
d.removed.append(brow.get(nxt.key))
|
|
137
|
+
|
|
138
|
+
d.added.sort(key=lambda r: _canonical_cell(r.get(d.key)))
|
|
139
|
+
d.changed.sort(key=lambda r: _canonical_cell(r.get(d.key)))
|
|
140
|
+
d.removed.sort(key=_canonical_cell)
|
|
141
|
+
return d
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _field_decl(fields: list[str], key: str) -> str:
|
|
145
|
+
return ",".join(("@" + format_key(f)) if f == key else format_key(f) for f in fields)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _encode_row(row: dict[str, Any], fields: list[str]) -> str:
|
|
149
|
+
return "|".join(format_scalar(row.get(f), "|") for f in fields)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def encode_generic_full(s: GenericSet, tool: str = "") -> str:
|
|
153
|
+
"""Emit a delta-participating full base payload: key= header, @id field, rows."""
|
|
154
|
+
name = s.name or "rows"
|
|
155
|
+
header = "GCF profile=generic"
|
|
156
|
+
if tool:
|
|
157
|
+
header += f" tool={tool}"
|
|
158
|
+
header += f" pack_root={generic_pack_root(s)} key={s.key}"
|
|
159
|
+
lines = [header, f"## {name} [{len(s.rows)}]{{{_field_decl(s.fields, s.key)}}}"]
|
|
160
|
+
lines += [_encode_row(row, s.fields) for row in s.rows]
|
|
161
|
+
return "\n".join(lines) + "\n"
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def encode_generic_delta(d: GenericDeltaPayload) -> str:
|
|
165
|
+
"""Serialize a delta payload (10a.2). Sections ordered added/changed/removed."""
|
|
166
|
+
header = "GCF profile=generic"
|
|
167
|
+
if d.tool:
|
|
168
|
+
header += f" tool={d.tool}"
|
|
169
|
+
header += f" delta=true base_root={d.base_root} new_root={d.new_root} key={d.key}"
|
|
170
|
+
if d.full_tokens > 0:
|
|
171
|
+
savings = 100.0 * (1.0 - d.delta_tokens / d.full_tokens)
|
|
172
|
+
header += f" savings={savings:.0f}%"
|
|
173
|
+
lines = [header]
|
|
174
|
+
if d.added:
|
|
175
|
+
lines.append(f"## added [{len(d.added)}]{{{_field_decl(d.fields, d.key)}}}")
|
|
176
|
+
lines += [_encode_row(r, d.fields) for r in d.added]
|
|
177
|
+
if d.changed:
|
|
178
|
+
lines.append(f"## changed [{len(d.changed)}]{{{_field_decl(d.fields, d.key)}}}")
|
|
179
|
+
lines += [_encode_row(r, d.fields) for r in d.changed]
|
|
180
|
+
if d.removed:
|
|
181
|
+
lines.append(f"## removed [{len(d.removed)}]{{@{d.key}}}")
|
|
182
|
+
lines += [format_scalar(idv, "|") for idv in d.removed]
|
|
183
|
+
return "\n".join(lines) + "\n"
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def verify_generic_delta(base: GenericSet, d: GenericDeltaPayload, expected_new_root: str) -> GenericSet:
|
|
187
|
+
"""Apply a delta to a base set and verify the result hashes to expected_new_root.
|
|
188
|
+
|
|
189
|
+
Atomic (10a.5): the whole payload is validated before any mutation; on failure the
|
|
190
|
+
base is left untouched and a ValueError is raised.
|
|
191
|
+
"""
|
|
192
|
+
if generic_pack_root(base) != d.base_root:
|
|
193
|
+
raise ValueError("base_mismatch: base root does not equal delta base_root")
|
|
194
|
+
base_by_id = _index_by_key(base)
|
|
195
|
+
|
|
196
|
+
for idv in d.removed:
|
|
197
|
+
if _canonical_cell(idv) not in base_by_id:
|
|
198
|
+
raise ValueError(f"delta_invalid: removing identity {_canonical_cell(idv)} not in base")
|
|
199
|
+
for row in d.added:
|
|
200
|
+
if _canonical_cell(row.get(d.key)) in base_by_id:
|
|
201
|
+
raise ValueError(f"delta_invalid: adding identity {_canonical_cell(row.get(d.key))} that already exists")
|
|
202
|
+
for row in d.changed:
|
|
203
|
+
if _canonical_cell(row.get(d.key)) not in base_by_id:
|
|
204
|
+
raise ValueError(f"delta_invalid: changing identity {_canonical_cell(row.get(d.key))} not in base")
|
|
205
|
+
|
|
206
|
+
work = dict(base_by_id)
|
|
207
|
+
for idv in d.removed:
|
|
208
|
+
work.pop(_canonical_cell(idv), None)
|
|
209
|
+
for row in d.added:
|
|
210
|
+
work[_canonical_cell(row.get(d.key))] = row
|
|
211
|
+
for row in d.changed:
|
|
212
|
+
work[_canonical_cell(row.get(d.key))] = row
|
|
213
|
+
|
|
214
|
+
result = GenericSet(key=base.key, fields=list(base.fields), rows=list(work.values()), name=base.name)
|
|
215
|
+
got = generic_pack_root(result)
|
|
216
|
+
if got != expected_new_root:
|
|
217
|
+
raise ValueError(f"root_mismatch: computed {got}, expected {expected_new_root}")
|
|
218
|
+
return result
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
# --- consumer-side wire parsing ---
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _parse_header_fields(header: str) -> dict[str, str]:
|
|
225
|
+
m: dict[str, str] = {}
|
|
226
|
+
for tok in header.split():
|
|
227
|
+
if "=" in tok:
|
|
228
|
+
k, _, v = tok.partition("=")
|
|
229
|
+
if k:
|
|
230
|
+
m[k] = v
|
|
231
|
+
return m
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _parse_count(s: str) -> int:
|
|
235
|
+
if s == "0":
|
|
236
|
+
return 0
|
|
237
|
+
if not s or s[0] == "0" or not s.isdigit():
|
|
238
|
+
raise ValueError(f"invalid_count: {s}")
|
|
239
|
+
return int(s)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _split_delta_field_decl(decl: str) -> tuple[list[str], str]:
|
|
243
|
+
if len(decl) < 2 or decl[0] != "{" or decl[-1] != "}":
|
|
244
|
+
raise ValueError(f"invalid field declaration: {decl}")
|
|
245
|
+
inner = decl[1:-1]
|
|
246
|
+
if inner == "":
|
|
247
|
+
return [], ""
|
|
248
|
+
fields: list[str] = []
|
|
249
|
+
key_field = ""
|
|
250
|
+
for raw in split_respecting_quotes(inner, ","):
|
|
251
|
+
f = raw.strip()
|
|
252
|
+
is_key = False
|
|
253
|
+
if f.startswith("@"):
|
|
254
|
+
f = f[1:]
|
|
255
|
+
is_key = True
|
|
256
|
+
if len(f) >= 2 and f[0] == '"' and f[-1] == '"':
|
|
257
|
+
from .scalar import parse_quoted_string
|
|
258
|
+
|
|
259
|
+
f = parse_quoted_string(f)
|
|
260
|
+
if is_key:
|
|
261
|
+
key_field = f
|
|
262
|
+
fields.append(f)
|
|
263
|
+
return fields, key_field
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _parse_section_header(content: str) -> tuple[str, int, list[str], str]:
|
|
267
|
+
bi = content.find(" [")
|
|
268
|
+
if bi < 0:
|
|
269
|
+
raise ValueError(f"delta_invalid: section header without count: {content!r}")
|
|
270
|
+
name = content[:bi].strip()
|
|
271
|
+
rest = content[bi + 1:] # "[N]{...}"
|
|
272
|
+
if not rest or rest[0] != "[":
|
|
273
|
+
raise ValueError(f"delta_invalid: malformed section header: {content!r}")
|
|
274
|
+
close = rest.find("]")
|
|
275
|
+
if close < 0:
|
|
276
|
+
raise ValueError(f"delta_invalid: unterminated count: {content!r}")
|
|
277
|
+
count = _parse_count(rest[1:close])
|
|
278
|
+
fields, key_field = _split_delta_field_decl(rest[close + 1:])
|
|
279
|
+
return name, count, fields, key_field
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _parse_row(line: str, fields: list[str]) -> dict[str, Any]:
|
|
283
|
+
cells = split_respecting_quotes(line, "|")
|
|
284
|
+
if len(cells) != len(fields):
|
|
285
|
+
raise ValueError(f"delta_invalid: row has {len(cells)} cells, expected {len(fields)}: {line!r}")
|
|
286
|
+
return {f: parse_scalar(cells[i], True) for i, f in enumerate(fields)}
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def decode_generic_full(text: str) -> tuple[GenericSet, str]:
|
|
290
|
+
"""Parse a delta-participating full base payload into (GenericSet, pack_root)."""
|
|
291
|
+
lines = text.rstrip("\n").split("\n")
|
|
292
|
+
if not lines:
|
|
293
|
+
raise ValueError("empty payload")
|
|
294
|
+
hdr = _parse_header_fields(lines[0])
|
|
295
|
+
if hdr.get("profile") != "generic":
|
|
296
|
+
raise ValueError("not a generic payload")
|
|
297
|
+
s = GenericSet(key=hdr.get("key", ""), fields=[], rows=[])
|
|
298
|
+
i = 1
|
|
299
|
+
while i < len(lines):
|
|
300
|
+
line = lines[i]
|
|
301
|
+
if not line.startswith("## "):
|
|
302
|
+
i += 1
|
|
303
|
+
continue
|
|
304
|
+
name, count, fields, key_field = _parse_section_header(line[3:])
|
|
305
|
+
s.name, s.fields = name, fields
|
|
306
|
+
if not s.key:
|
|
307
|
+
s.key = key_field
|
|
308
|
+
i += 1
|
|
309
|
+
for _ in range(count):
|
|
310
|
+
if i >= len(lines):
|
|
311
|
+
raise ValueError("delta_invalid: fewer rows than declared count")
|
|
312
|
+
s.rows.append(_parse_row(lines[i], fields))
|
|
313
|
+
i += 1
|
|
314
|
+
return s, hdr.get("pack_root", "")
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def decode_generic_delta(text: str) -> GenericDeltaPayload:
|
|
318
|
+
"""Parse a delta payload (10a.2) into a GenericDeltaPayload for application."""
|
|
319
|
+
lines = text.rstrip("\n").split("\n")
|
|
320
|
+
if not lines:
|
|
321
|
+
raise ValueError("empty payload")
|
|
322
|
+
hdr = _parse_header_fields(lines[0])
|
|
323
|
+
if hdr.get("profile") != "generic":
|
|
324
|
+
raise ValueError("not a generic payload")
|
|
325
|
+
if hdr.get("delta") != "true":
|
|
326
|
+
raise ValueError("not a delta payload")
|
|
327
|
+
d = GenericDeltaPayload(
|
|
328
|
+
key=hdr.get("key", ""),
|
|
329
|
+
fields=[],
|
|
330
|
+
base_root=hdr.get("base_root", ""),
|
|
331
|
+
new_root=hdr.get("new_root", ""),
|
|
332
|
+
tool=hdr.get("tool", ""),
|
|
333
|
+
)
|
|
334
|
+
i = 1
|
|
335
|
+
while i < len(lines):
|
|
336
|
+
line = lines[i]
|
|
337
|
+
if not line.startswith("## "):
|
|
338
|
+
i += 1
|
|
339
|
+
continue
|
|
340
|
+
name, count, fields, key_field = _parse_section_header(line[3:])
|
|
341
|
+
if not d.key and key_field:
|
|
342
|
+
d.key = key_field
|
|
343
|
+
if not d.fields and name in ("added", "changed"):
|
|
344
|
+
d.fields = fields
|
|
345
|
+
i += 1
|
|
346
|
+
if name in ("added", "changed"):
|
|
347
|
+
rows = []
|
|
348
|
+
for _ in range(count):
|
|
349
|
+
if i >= len(lines):
|
|
350
|
+
raise ValueError(f"delta_invalid: fewer rows than declared count in ## {name}")
|
|
351
|
+
rows.append(_parse_row(lines[i], fields))
|
|
352
|
+
i += 1
|
|
353
|
+
if name == "added":
|
|
354
|
+
d.added = rows
|
|
355
|
+
else:
|
|
356
|
+
d.changed = rows
|
|
357
|
+
elif name == "removed":
|
|
358
|
+
for _ in range(count):
|
|
359
|
+
if i >= len(lines):
|
|
360
|
+
raise ValueError("delta_invalid: fewer identities than declared count in ## removed")
|
|
361
|
+
d.removed.append(parse_scalar(lines[i], True))
|
|
362
|
+
i += 1
|
|
363
|
+
else:
|
|
364
|
+
raise ValueError(f"delta_invalid: unknown delta section {name!r}")
|
|
365
|
+
return d
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
# --- producer-side re-anchor session helper (SPEC Section 10a.8) ---
|
|
369
|
+
#
|
|
370
|
+
# GenericDeltaSession manages the re-anchor cadence for a stream of
|
|
371
|
+
# generic-profile updates. It is thin sugar over the primitives: each ``next``
|
|
372
|
+
# emits either a compact delta or, on its chosen cadence, a full re-anchor,
|
|
373
|
+
# updating its held base. It introduces NO new wire syntax: every payload it
|
|
374
|
+
# emits is exactly what ``encode_generic_full`` or ``encode_generic_delta``
|
|
375
|
+
# produce, and the decoder accepts them cadence-agnostically. N and the size
|
|
376
|
+
# guard are the helper's knobs; they are never wire fields. (Section 10a.8 is
|
|
377
|
+
# non-normative producer policy.)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
# The working default cadence for FixedN (SPEC Section 10a.8).
|
|
381
|
+
DEFAULT_REANCHOR_N = 15
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
class ReanchorMode(Enum):
|
|
385
|
+
"""Selects the session's cadence policy."""
|
|
386
|
+
|
|
387
|
+
FIXED_N = 0 # re-anchor every N turns
|
|
388
|
+
SIZE_GUARD = 1 # re-anchor once cumulative delta reaches the full payload's size
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
@dataclass
|
|
392
|
+
class ReanchorPolicy:
|
|
393
|
+
"""Selects when a :class:`GenericDeltaSession` re-anchors.
|
|
394
|
+
|
|
395
|
+
Construct it with :func:`fixed_n` or :func:`size_guard`.
|
|
396
|
+
"""
|
|
397
|
+
|
|
398
|
+
mode: ReanchorMode = ReanchorMode.FIXED_N
|
|
399
|
+
n: int = 0 # turns between anchors; FixedN only
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def fixed_n(n: int) -> ReanchorPolicy:
|
|
403
|
+
"""Re-anchor every ``n`` turns. ``n <= 0`` falls back to ``DEFAULT_REANCHOR_N``."""
|
|
404
|
+
if n <= 0:
|
|
405
|
+
n = DEFAULT_REANCHOR_N
|
|
406
|
+
return ReanchorPolicy(mode=ReanchorMode.FIXED_N, n=n)
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def size_guard() -> ReanchorPolicy:
|
|
410
|
+
"""Re-anchor once the cumulative delta bytes since the last anchor reach the
|
|
411
|
+
current full payload's byte size: it re-anchors more under heavy churn, rarely
|
|
412
|
+
under light churn, and bounds the delta spent between anchors to about one full
|
|
413
|
+
payload. Production-recommended.
|
|
414
|
+
"""
|
|
415
|
+
return ReanchorPolicy(mode=ReanchorMode.SIZE_GUARD)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _byte_len(s: str) -> int:
|
|
419
|
+
"""UTF-8 byte length, matching Go's ``len(string)`` (bytes, not code points)."""
|
|
420
|
+
return len(s.encode("utf-8"))
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
class GenericDeltaSession:
|
|
424
|
+
"""Holds the current base and re-anchor state for a producer loop.
|
|
425
|
+
|
|
426
|
+
Not safe for concurrent use. Call :meth:`current_full` to get the initial
|
|
427
|
+
full payload to transmit, then :meth:`next` for each subsequent state.
|
|
428
|
+
"""
|
|
429
|
+
|
|
430
|
+
def __init__(self, base: GenericSet, tool: str, policy: ReanchorPolicy) -> None:
|
|
431
|
+
if policy.mode == ReanchorMode.FIXED_N and policy.n <= 0:
|
|
432
|
+
policy = ReanchorPolicy(mode=policy.mode, n=DEFAULT_REANCHOR_N)
|
|
433
|
+
self._base = base
|
|
434
|
+
self._tool = tool
|
|
435
|
+
self._policy = policy
|
|
436
|
+
self._turn = 0
|
|
437
|
+
self._cum = 0 # cumulative delta bytes since the last anchor
|
|
438
|
+
|
|
439
|
+
@property
|
|
440
|
+
def turn(self) -> int:
|
|
441
|
+
"""The number of :meth:`next` calls so far (the initial full is turn 0)."""
|
|
442
|
+
return self._turn
|
|
443
|
+
|
|
444
|
+
def current_full(self) -> str:
|
|
445
|
+
"""Return the full payload for the current base (:func:`encode_generic_full`).
|
|
446
|
+
|
|
447
|
+
Send this first to establish the base; it is also a valid manual re-anchor.
|
|
448
|
+
"""
|
|
449
|
+
return encode_generic_full(self._base, self._tool)
|
|
450
|
+
|
|
451
|
+
def _reanchor(self, nxt: GenericSet) -> str:
|
|
452
|
+
wire = encode_generic_full(nxt, self._tool)
|
|
453
|
+
self._base = nxt
|
|
454
|
+
self._cum = 0
|
|
455
|
+
return wire
|
|
456
|
+
|
|
457
|
+
def next(self, nxt: GenericSet) -> tuple[str, bool]:
|
|
458
|
+
"""Advance the session by one turn to ``nxt``.
|
|
459
|
+
|
|
460
|
+
Returns ``(wire, is_full)``: the wire to transmit and whether it is a full
|
|
461
|
+
re-anchor (``True``) or a delta (``False``). A schema change forces a full
|
|
462
|
+
(Section 10a.7). The held base becomes ``nxt`` either way. The wire is
|
|
463
|
+
byte-identical to calling ``encode_generic_full`` / ``encode_generic_delta``
|
|
464
|
+
directly. May raise on duplicate identity / no key (propagated from
|
|
465
|
+
:func:`diff_generic_sets`).
|
|
466
|
+
"""
|
|
467
|
+
self._turn += 1
|
|
468
|
+
|
|
469
|
+
# Schema change (or a fresh key) cannot be expressed as a delta -> full.
|
|
470
|
+
if nxt.key != self._base.key or list(self._base.fields) != list(nxt.fields):
|
|
471
|
+
return self._reanchor(nxt), True
|
|
472
|
+
|
|
473
|
+
d = diff_generic_sets(self._base, nxt)
|
|
474
|
+
delta_wire = encode_generic_delta(d)
|
|
475
|
+
|
|
476
|
+
if self._policy.mode == ReanchorMode.SIZE_GUARD:
|
|
477
|
+
reanchor = self._cum + _byte_len(delta_wire) >= _byte_len(
|
|
478
|
+
encode_generic_full(nxt, self._tool)
|
|
479
|
+
)
|
|
480
|
+
else: # FIXED_N
|
|
481
|
+
reanchor = self._turn % self._policy.n == 0
|
|
482
|
+
|
|
483
|
+
if reanchor:
|
|
484
|
+
return self._reanchor(nxt), True
|
|
485
|
+
self._base = nxt
|
|
486
|
+
self._cum += _byte_len(delta_wire)
|
|
487
|
+
return delta_wire, False
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gcf-python
|
|
3
|
-
Version: 2.
|
|
4
|
-
Summary: The AI-native wire format for structured data. 50-92% fewer tokens than JSON. 100% comprehension on every frontier model. Zero dependencies.
|
|
3
|
+
Version: 2.3.0
|
|
4
|
+
Summary: The AI-native wire format for structured data. 50-92% fewer tokens than JSON, with multi-turn delta encoding for agent loops. 100% comprehension on every frontier model. Zero dependencies.
|
|
5
5
|
Project-URL: Homepage, https://github.com/blackwell-systems/gcf-python
|
|
6
|
-
Project-URL: Documentation, https://
|
|
6
|
+
Project-URL: Documentation, https://gcformat.com/
|
|
7
7
|
Project-URL: Specification, https://github.com/blackwell-systems/gcf
|
|
8
8
|
Author: Blackwell Systems
|
|
9
9
|
License: MIT
|
|
10
10
|
License-File: LICENSE
|
|
11
|
-
Keywords: gcf,graph,llm,mcp,token-efficient,wire-format
|
|
12
|
-
Classifier: Development Status ::
|
|
11
|
+
Keywords: agents,delta,gcf,graph,llm,mcp,multi-turn,token-efficient,wire-format
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
13
|
Classifier: Intended Audience :: Developers
|
|
14
14
|
Classifier: License :: OSI Approved :: MIT License
|
|
15
15
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -182,6 +182,44 @@ Output:
|
|
|
182
182
|
|
|
183
183
|
Works on dicts, lists, and primitives. Lists of uniform dicts get tabular rows. Nested dicts use `## key` section headers.
|
|
184
184
|
|
|
185
|
+
## Generic-Profile Delta (multi-turn)
|
|
186
|
+
|
|
187
|
+
In an agent loop the same keyed table gets re-queried turn after turn. Instead of re-sending the whole table each time, send only the changed rows (SPEC §10a):
|
|
188
|
+
|
|
189
|
+
```python
|
|
190
|
+
from gcf import GenericSet, diff_generic_sets, encode_generic_delta, verify_generic_delta
|
|
191
|
+
|
|
192
|
+
base = GenericSet(key="id", fields=["id", "status"], rows=[
|
|
193
|
+
{"id": 1001, "status": "pending"},
|
|
194
|
+
{"id": 1002, "status": "shipped"},
|
|
195
|
+
])
|
|
196
|
+
nxt = GenericSet(key="id", fields=["id", "status"], rows=[
|
|
197
|
+
{"id": 1001, "status": "shipped"}, # changed
|
|
198
|
+
{"id": 1003, "status": "pending"}, # added (1002 removed)
|
|
199
|
+
])
|
|
200
|
+
|
|
201
|
+
d = diff_generic_sets(base, nxt)
|
|
202
|
+
wire = encode_generic_delta(d) # ## added / ## changed / ## removed
|
|
203
|
+
held = verify_generic_delta(base, d, d.new_root) # atomic apply + new_root verification
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Opt-in and bilateral, keyed on content-addressed pack roots. By the 5th overlapping call, ~97% fewer tokens than re-sending JSON.
|
|
207
|
+
|
|
208
|
+
### Re-anchor session helper
|
|
209
|
+
|
|
210
|
+
`GenericDeltaSession` manages the delta/re-anchor cadence for you: each `next()` returns either a compact delta or, on its cadence, a full re-anchor (which re-grounds the consumer), updating its held base.
|
|
211
|
+
|
|
212
|
+
```python
|
|
213
|
+
from gcf import GenericDeltaSession, fixed_n, size_guard
|
|
214
|
+
|
|
215
|
+
sess = GenericDeltaSession(base, tool="orders", policy=size_guard())
|
|
216
|
+
wire = sess.current_full() # transmit the base once to establish it
|
|
217
|
+
for snapshot in stream: # each turn's current GenericSet
|
|
218
|
+
wire, is_full = sess.next(snapshot) # a compact delta, or a periodic full re-anchor
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
`fixed_n(15)` re-anchors every N turns; `size_guard()` (recommended) re-anchors once the cumulative delta reaches a full payload's size. It introduces no new wire syntax and the decoder stays cadence-agnostic, so a re-anchor is just the protocol's "full" outcome on a schedule.
|
|
222
|
+
|
|
185
223
|
## API
|
|
186
224
|
|
|
187
225
|
| Function | Description |
|
|
@@ -190,7 +228,11 @@ Works on dicts, lists, and primitives. Lists of uniform dicts get tabular rows.
|
|
|
190
228
|
| `encode_generic(data: Any) -> str` | Encode any value to GCF tabular format |
|
|
191
229
|
| `decode(input_text: str) -> Payload` | Parse GCF text back to a Payload |
|
|
192
230
|
| `encode_with_session(p: Payload, s: Session) -> str` | Encode with session deduplication |
|
|
193
|
-
| `encode_delta(d: DeltaPayload) -> str` | Encode a delta (added/removed only) |
|
|
231
|
+
| `encode_delta(d: DeltaPayload) -> str` | Encode a graph delta (added/removed only) |
|
|
232
|
+
| `diff_generic_sets(base, next) -> GenericDeltaPayload` | Diff two keyed record sets (generic profile) |
|
|
233
|
+
| `encode_generic_delta(d) -> str` / `decode_generic_delta(s)` | Generic-profile delta wire (§10a) |
|
|
234
|
+
| `verify_generic_delta(base, d, root) -> GenericSet` | Atomic apply + `new_root` verification |
|
|
235
|
+
| `GenericDeltaSession(base, tool, policy)` | Producer-side re-anchor cadence helper (§10a.8) |
|
|
194
236
|
| `Session()` | Create a new session tracker (thread-safe) |
|
|
195
237
|
|
|
196
238
|
## Types
|
|
@@ -200,7 +242,9 @@ Works on dicts, lists, and primitives. Lists of uniform dicts get tabular rows.
|
|
|
200
242
|
| `Payload` | Full GCF payload: tool, budget, symbols, edges, pack root |
|
|
201
243
|
| `Symbol` | Graph node: qualified name, kind, score, provenance, distance |
|
|
202
244
|
| `Edge` | Directed relationship: source, target, edge type |
|
|
203
|
-
| `DeltaPayload` | Diff between two packs: added/removed symbols and edges |
|
|
245
|
+
| `DeltaPayload` | Diff between two graph packs: added/removed symbols and edges |
|
|
246
|
+
| `GenericSet` / `GenericDeltaPayload` | Keyed record set and its generic-profile diff (§10a) |
|
|
247
|
+
| `GenericDeltaSession` | Stateful producer that schedules delta vs full re-anchor (§10a.8) |
|
|
204
248
|
| `Session` | Thread-safe tracker for multi-call deduplication |
|
|
205
249
|
| `KIND_ABBREV` / `KIND_EXPAND` | Bidirectional kind abbreviation dicts |
|
|
206
250
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
gcf/__init__.py,sha256=
|
|
1
|
+
gcf/__init__.py,sha256=WtDoZGghWL_LuNRGmB1uyfxQAs7zOAAMnwwfJK4vd-Y,2499
|
|
2
2
|
gcf/__main__.py,sha256=EpvBz1yc8H0D5OJ1zy2tYke-kRzvudKa4DEbfeW14ao,71
|
|
3
3
|
gcf/cli.py,sha256=UEe1CAZn-rKGNIo_ap8-oez3ucl6DSRbsdv6RDnzygY,5256
|
|
4
4
|
gcf/constants.py,sha256=cmZ8YJSOB0im_eyfN8v4UvrLpBC6Fuf4cfcKZGbutxY,638
|
|
@@ -7,13 +7,14 @@ gcf/decode_generic.py,sha256=ZlFZqwenOzCnvbJijKscb6aQHoHQ81qoHBkaalVbqqo,26381
|
|
|
7
7
|
gcf/delta.py,sha256=f90UC6zejXH-ujyU_OViWDCqnxLXk3i6Qion-RJGHY4,1670
|
|
8
8
|
gcf/encode.py,sha256=CcqMSNmojrQAAw2X3MNfw8YR6l4rNw9hWDqwmf846oA,2929
|
|
9
9
|
gcf/generic.py,sha256=t0Px1fAoWsmFFeICbL65LL8UTGFfnbh4PCanvMMKqYs,18198
|
|
10
|
+
gcf/generic_delta.py,sha256=GU7CNumz3PQhC0Xs5pO30Nb3ACvSorHEQMu-4kazpFM,17838
|
|
10
11
|
gcf/scalar.py,sha256=MZay-KIROvaFHet-g2-pBghahT1bf_5bxZjG4yTZkSo,9329
|
|
11
12
|
gcf/session.py,sha256=6-wytNGaBkgqH7uTar7ojUFYNwIMeEVAZhuV-s-upZM,4704
|
|
12
13
|
gcf/stream.py,sha256=1Kt_a2daKpYHlWP1lnDJ4g549pUt9rc-U0xbARbXRKQ,5174
|
|
13
14
|
gcf/stream_generic.py,sha256=RnqqiPSu5joJa-7e58QbbzSGfvxBICA587A3aArjZvE,3250
|
|
14
15
|
gcf/types.py,sha256=AWm-LQoSqLHAYtEjcAxWQZqJ4JXqNreLUKO2mJFgNMA,1465
|
|
15
|
-
gcf_python-2.
|
|
16
|
-
gcf_python-2.
|
|
17
|
-
gcf_python-2.
|
|
18
|
-
gcf_python-2.
|
|
19
|
-
gcf_python-2.
|
|
16
|
+
gcf_python-2.3.0.dist-info/METADATA,sha256=IVEaubFtiRRpdcLqbYoPoibNEu-ueoaW--v9g_PhFpE,13040
|
|
17
|
+
gcf_python-2.3.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
18
|
+
gcf_python-2.3.0.dist-info/entry_points.txt,sha256=aFT6gqlkh8iGfM8cblE-LUMxHH08_v71IIoZtDdRIVA,37
|
|
19
|
+
gcf_python-2.3.0.dist-info/licenses/LICENSE,sha256=2Fit9wnaIe--RMSAgyQqxC5hfZTyZqn4fIdBtp9qPDw,1072
|
|
20
|
+
gcf_python-2.3.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|