gcf-python 0.3.1__py3-none-any.whl → 0.5.0__py3-none-any.whl

gcf/__init__.py

@@ -40,19 +40,25 @@ from .delta import encode_delta
 from .encode import encode
 from .generic import encode_generic
 from .session import Session, encode_with_session
+from .decode_generic import decode_generic
+from .stream import StreamEncoder
+from .stream_generic import GenericStreamEncoder
 from .types import Components, DeltaPayload, Edge, Payload, Symbol
 
 __all__ = [
     "Components",
     "DecodeError",
     "DeltaPayload",
+    "GenericStreamEncoder",
     "Edge",
     "KIND_ABBREV",
     "KIND_EXPAND",
     "Payload",
     "Session",
+    "StreamEncoder",
     "Symbol",
     "decode",
+    "decode_generic",
     "encode",
     "encode_delta",
     "encode_generic",

gcf/decode_generic.py

@@ -0,0 +1,255 @@
+"""GCF generic decoder: parses any GCF text (tabular or graph) back to Python objects."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .decode import decode
+
+
+def decode_generic(input_text: str) -> Any:
+    """Decode any GCF text back into Python objects.
+
+    Handles tabular arrays, key-value pairs, nested sections, inline
+    primitive arrays, and graph profile payloads.
+
+    Returns dicts, lists, and primitives matching the original structure.
+    """
+    input_text = input_text.rstrip("\n\r")
+    if not input_text:
+        return None
+
+    lines = input_text.split("\n")
+
+    # Graph profile fallback.
+    if lines[0].startswith("GCF "):
+        p = decode(input_text)
+        return {
+            "tool": p.tool,
+            "tokenBudget": p.token_budget,
+            "tokensUsed": p.tokens_used,
+            "packRoot": p.pack_root,
+            "symbols": [
+                {
+                    "qualifiedName": s.qualified_name,
+                    "kind": s.kind,
+                    "score": s.score,
+                    "provenance": s.provenance,
+                    "distance": s.distance,
+                }
+                for s in p.symbols
+            ],
+            "edges": [
+                {
+                    "source": e.source,
+                    "target": e.target,
+                    "edgeType": e.edge_type,
+                    **({"status": e.status} if e.status else {}),
+                }
+                for e in p.edges
+            ],
+        }
+
+    result: dict[str, Any] = {}
+    _parse_object(lines, 0, 0, result)
+    return result
+
+
+def _parse_object(lines: list[str], start: int, depth: int, out: dict[str, Any]) -> int:
+    indent = "  " * depth
+    i = start
+
+    while i < len(lines):
+        raw = lines[i].rstrip("\r")
+        if raw == "" or raw.startswith("# "):
+            i += 1
+            continue
+
+        if depth > 0 and not raw.startswith(indent):
+            break
+
+        content = raw[len(indent):] if depth > 0 else raw
+
+        if content.startswith("## _summary"):
+            i += 1
+            continue
+
+        if content.startswith("## "):
+            header = content[3:]
+            bracket_idx = header.find(" [")
+
+            if bracket_idx >= 0:
+                name = header[:bracket_idx]
+                rest = header[bracket_idx + 2:]
+                close_bracket = rest.find("]")
+
+                if close_bracket >= 0:
+                    after_bracket = rest[close_bracket + 1:]
+
+                    if after_bracket.startswith("{"):
+                        field_end = after_bracket.find("}")
+                        if field_end >= 0:
+                            fields = after_bracket[1:field_end].split(",")
+                            i += 1
+                            rows, consumed = _parse_tabular_rows(lines, i, depth, fields)
+                            out[name] = rows
+                            i += consumed
+                            continue
+                    else:
+                        count_str = rest[:close_bracket]
+                        if count_str == "0":
+                            out[name] = []
+                            i += 1
+                            continue
+                        i += 1
+                        items, consumed = _parse_non_uniform_array(lines, i, depth)
+                        out[name] = items
+                        i += consumed
+                        continue
+
+            name = header
+            bi = name.find(" [")
+            if bi >= 0:
+                name = name[:bi]
+            i += 1
+            nested: dict[str, Any] = {}
+            consumed = _parse_object(lines, i, depth + 1, nested)
+            out[name] = nested
+            i += consumed
+            continue
+
+        # Inline primitive array.
+        bracket_idx = content.find("[")
+        if bracket_idx > 0:
+            colon_idx = content.find("]: ")
+            if colon_idx > bracket_idx:
+                name = content[:bracket_idx]
+                vals_str = content[colon_idx + 3:]
+                out[name] = [_parse_value(v.strip()) for v in vals_str.split(",")]
+                i += 1
+                continue
+
+        # Key=value.
+        eq_idx = content.find("=")
+        if eq_idx > 0:
+            key = content[:eq_idx]
+            val = content[eq_idx + 1:]
+            out[key] = _parse_value(val)
+            i += 1
+            continue
+
+        i += 1
+
+    return i - start
+
+
+def _parse_tabular_rows(
+    lines: list[str], start: int, depth: int, fields: list[str]
+) -> tuple[list[Any], int]:
+    indent = "  " * depth
+    rows: list[Any] = []
+    i = start
+
+    while i < len(lines):
+        raw = lines[i].rstrip("\r")
+        if raw == "":
+            i += 1
+            continue
+
+        if depth > 0 and not raw.startswith(indent):
+            break
+        content = raw[len(indent):] if depth > 0 else raw
+
+        if content.startswith("## "):
+            break
+        if content.startswith("# "):
+            i += 1
+            continue
+
+        row_data = content
+        has_nested = False
+        if row_data.startswith("@"):
+            sp = row_data.find(" ")
+            if sp > 0:
+                row_data = row_data[sp + 1:]
+                has_nested = True
+
+        vals = row_data.split("|")
+        row: dict[str, Any] = {}
+        for j, f in enumerate(fields):
+            row[f] = _parse_value(vals[j]) if j < len(vals) else None
+
+        i += 1
+
+        if has_nested:
+            nested_indent = indent + "  "
+            while i < len(lines):
+                nl = lines[i].rstrip("\r")
+                if not nl.startswith(nested_indent):
+                    break
+                nc = nl[len(nested_indent):]
+
+                if nc.startswith("."):
+                    field_name = nc[1:]
+                    i += 1
+                    nested: dict[str, Any] = {}
+                    consumed = _parse_object(lines, i, depth + 2, nested)
+                    row[field_name] = nested
+                    i += consumed
+                else:
+                    break
+
+        rows.append(row)
+
+    return rows, i - start
+
+
+def _parse_non_uniform_array(
+    lines: list[str], start: int, depth: int
+) -> tuple[list[Any], int]:
+    indent = "  " * depth
+    items: list[Any] = []
+    i = start
+
+    while i < len(lines):
+        raw = lines[i].rstrip("\r")
+        if raw == "":
+            i += 1
+            continue
+        if depth > 0 and not raw.startswith(indent):
+            break
+        content = raw[len(indent):] if depth > 0 else raw
+        if content.startswith("## "):
+            break
+
+        if content.startswith("@"):
+            sp = content.find(" ")
+            if sp > 0:
+                items.append(_parse_value(content[sp + 1:]))
+            i += 1
+        else:
+            break
+
+    return items, i - start
+
+
+def _parse_value(s: str) -> Any:
+    if s == "-":
+        return None
+    if s == "true":
+        return True
+    if s == "false":
+        return False
+    if s == '""':
+        return ""
+    if len(s) >= 2 and s[0] == '"' and s[-1] == '"':
+        return s[1:-1].replace('\\"', '"').replace("\\\\", "\\")
+    try:
+        return int(s)
+    except ValueError:
+        pass
+    try:
+        return float(s)
+    except ValueError:
+        pass
+    return s

gcf/stream.py

@@ -0,0 +1,151 @@
+"""GCF streaming encoder: zero-buffering encode to any writable."""
+
+from __future__ import annotations
+
+import threading
+from typing import Any, Protocol
+
+from .constants import KIND_ABBREV
+from .types import Edge, Symbol
+
+
+class StreamWriter(Protocol):
+    """Any object with a write(s: str) method."""
+
+    def write(self, s: str) -> Any: ...
+
+
+class StreamEncoder:
+    """Writes GCF output incrementally as symbols and edges arrive.
+
+    Zero buffering: each symbol/edge is written immediately. A trailer summary
+    is emitted on close() with the final counts.
+
+    Example::
+
+        enc = StreamEncoder(sys.stdout, "context_for_task", token_budget=5000)
+        enc.write_symbol(sym1)  # emitted immediately
+        enc.write_edge(edge1)   # emitted immediately
+        enc.close()             # emits ## _summary trailer
+    """
+
+    def __init__(
+        self,
+        writer: StreamWriter,
+        tool: str,
+        *,
+        token_budget: int = 0,
+        tokens_used: int = 0,
+        pack_root: str = "",
+        session: bool = False,
+    ) -> None:
+        self._w = writer
+        self._lock = threading.Lock()
+        self._sym_index: dict[str, int] = {}
+        self._next_id = 0
+        self._current_group = ""
+        self._group_counts: dict[str, int] = {}
+        self._edge_count = 0
+        self._edges_started = False
+
+        # Emit header immediately.
+        parts = [f"GCF tool={tool}"]
+        if token_budget:
+            parts.append(f"budget={token_budget}")
+        if tokens_used:
+            parts.append(f"tokens={tokens_used}")
+        if pack_root:
+            parts.append(f"pack_root={pack_root}")
+        if session:
+            parts.append("session=true")
+        self._w.write(" ".join(parts) + "\n")
+
+    def write_symbol(self, s: Symbol) -> None:
+        """Emit a symbol line immediately. Group headers auto-managed."""
+        with self._lock:
+            group_names = ["targets", "related", "extended"]
+            if s.distance < len(group_names):
+                group_name = group_names[s.distance]
+            else:
+                group_name = f"distance_{s.distance}"
+
+            if group_name != self._current_group:
+                self._w.write(f"## {group_name}\n")
+                self._current_group = group_name
+
+            idx = self._next_id
+            self._sym_index[s.qualified_name] = idx
+            self._next_id += 1
+
+            kind = KIND_ABBREV.get(s.kind, s.kind)
+            self._w.write(f"@{idx} {kind} {s.qualified_name} {s.score:.2f} {s.provenance}\n")
+
+            self._group_counts[group_name] = self._group_counts.get(group_name, 0) + 1
+
+    def write_edge(self, e: Edge) -> None:
+        """Emit an edge line immediately. Edges section header auto-emitted on first edge."""
+        with self._lock:
+            src_idx = self._sym_index.get(e.source)
+            tgt_idx = self._sym_index.get(e.target)
+            if src_idx is None or tgt_idx is None:
+                return
+
+            if not self._edges_started:
+                self._w.write("## edges [?]\n")
+                self._edges_started = True
+
+            line = f"@{tgt_idx}<@{src_idx} {e.edge_type}"
+            if e.status and e.status != "unchanged":
+                line += f" {e.status}"
+            self._w.write(line + "\n")
+            self._edge_count += 1
+
+    def write_bare_ref(self, qname: str, distance: int) -> None:
+        """Emit a bare reference for a previously-transmitted symbol (session mode)."""
+        with self._lock:
+            group_names = ["targets", "related", "extended"]
+            if distance < len(group_names):
+                group_name = group_names[distance]
+            else:
+                group_name = f"distance_{distance}"
+
+            if group_name != self._current_group:
+                self._w.write(f"## {group_name}\n")
+                self._current_group = group_name
+
+            idx = self._next_id
+            self._sym_index[qname] = idx
+            self._next_id += 1
+            self._w.write(f"@{idx}  # previously transmitted\n")
+            self._group_counts[group_name] = self._group_counts.get(group_name, 0) + 1
+
+    def close(self) -> None:
+        """Emit ## _summary trailer with final counts."""
+        with self._lock:
+            sections: list[str] = []
+            group_order = ["targets", "related", "extended"]
+
+            for g in group_order:
+                c = self._group_counts.get(g, 0)
+                if c > 0:
+                    sections.append(f"{g}:{c}")
+            for g, c in self._group_counts.items():
+                if g not in group_order and c > 0:
+                    sections.append(f"{g}:{c}")
+            if self._edge_count > 0:
+                sections.append(f"edges:{self._edge_count}")
+
+            self._w.write(
+                f"## _summary symbols={self._next_id} edges={self._edge_count}"
+                f" sections={','.join(sections)}\n"
+            )
+
+    @property
+    def symbol_count(self) -> int:
+        """Number of symbols written so far."""
+        return self._next_id
+
+    @property
+    def edge_count(self) -> int:
+        """Number of edges written so far."""
+        return self._edge_count

gcf/stream_generic.py

@@ -0,0 +1,111 @@
+"""GCF generic streaming encoder: zero-buffering tabular encode to any writable."""
+
+from __future__ import annotations
+
+import threading
+from typing import Any, Sequence
+
+
+class GenericStreamEncoder:
+    """Writes GCF tabular output incrementally as rows arrive.
+
+    Zero buffering: each row is written immediately. A trailer summary is
+    emitted on close() with the final counts.
+
+    Example::
+
+        enc = GenericStreamEncoder(sys.stdout)
+        enc.begin_array("employees", ["id", "name", "department", "salary"])
+        enc.write_row([1, "Alice", "Engineering", 95000])
+        enc.write_row([2, "Bob", "Sales", 72000])
+        enc.end_array()
+        enc.close()
+    """
+
+    def __init__(self, writer: Any) -> None:
+        self._w = writer
+        self._lock = threading.Lock()
+        self._sections: list[tuple[str, int]] = []
+        self._current: dict[str, Any] | None = None
+
+    def begin_array(self, name: str, fields: Sequence[str]) -> None:
+        """Start a tabular array section with deferred count [?]."""
+        with self._lock:
+            if self._current is not None:
+                self._end_array_locked()
+            self._w.write(f"## {name} [?]{{{','.join(fields)}}}\n")
+            self._current = {"name": name, "fields": list(fields), "count": 0}
+
+    def write_row(self, values: Sequence[Any]) -> None:
+        """Emit a single pipe-separated row immediately."""
+        with self._lock:
+            if self._current is None:
+                return
+            parts = [_format_value(v) for v in values]
+            self._w.write("|".join(parts) + "\n")
+            self._current["count"] += 1
+
+    def end_array(self) -> None:
+        """Close the current array section and record its count."""
+        with self._lock:
+            self._end_array_locked()
+
+    def write_kv(self, key: str, value: Any) -> None:
+        """Emit a key=value line immediately."""
+        with self._lock:
+            self._w.write(f"{key}={_format_value(value)}\n")
+
+    def write_section(self, name: str) -> None:
+        """Start a nested object section (## key)."""
+        with self._lock:
+            if self._current is not None:
+                self._end_array_locked()
+            self._w.write(f"## {name}\n")
+
+    def write_inline_array(self, name: str, values: Sequence[Any]) -> None:
+        """Emit a primitive array inline: name[N]: val1,val2,val3"""
+        with self._lock:
+            parts = [_format_value(v) for v in values]
+            self._w.write(f"{name}[{len(values)}]: {','.join(parts)}\n")
+
+    def close(self) -> None:
+        """Emit the ## _summary trailer with final counts."""
+        with self._lock:
+            if self._current is not None:
+                self._end_array_locked()
+            if not self._sections:
+                return
+            total_rows = 0
+            section_parts: list[str] = []
+            for name, count in self._sections:
+                section_parts.append(f"{name}:{count}")
+                total_rows += count
+            self._w.write(
+                f"## _summary rows={total_rows} sections={','.join(section_parts)}\n"
+            )
+
+    def _end_array_locked(self) -> None:
+        if self._current is None:
+            return
+        self._sections.append((self._current["name"], self._current["count"]))
+        self._current = None
+
+
+def _format_value(v: Any) -> str:
+    if v is None:
+        return "-"
+    if isinstance(v, bool):
+        return "true" if v else "false"
+    if isinstance(v, int):
+        return str(v)
+    if isinstance(v, float):
+        # Match Go's %g formatting
+        s = f"{v:g}"
+        return s
+    if isinstance(v, str):
+        if v == "":
+            return '""'
+        if "|" in v or "\n" in v:
+            return '"' + v.replace('"', '\\"') + '"'
+        return v
+    return str(v)

{gcf_python-0.3.1.dist-info → gcf_python-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gcf-python
-Version: 0.3.1
+Version: 0.5.0
 Summary: Python implementation of GCF (Graph Compact Format): token-optimized wire format for LLM tool responses
 Project-URL: Homepage, https://github.com/blackwell-systems/gcf-python
 Project-URL: Documentation, https://blackwell-systems.github.io/gcf/
@@ -30,7 +30,7 @@ Description-Content-Type: text/markdown
 
 # gcf-python
 
-Python implementation of [GCF (Graph Compact Format)](https://gcformat.com/) — the most token-efficient wire format for LLMs. A drop-in alternative to JSON and TOON for any structured data.
+Python implementation of [GCF](https://gcformat.com/) — the most token-efficient wire format for LLMs. A drop-in alternative to JSON and TOON for any structured data.
 
 **79% fewer input tokens than JSON. 75% fewer output tokens. 52% smaller than TOON. 100% LLM comprehension at 500 symbols, where JSON scores 76.9% and TOON scores 92.3%.**
 
@@ -66,33 +66,21 @@ Payload: 50 symbols, 20 edges
 ### Quick Start
 
 ```python
-from gcf import encode, Payload, Symbol, Edge
+from gcf import encode_generic
 
-p = Payload(
-    tool="context_for_task",
-    token_budget=5000,
-    tokens_used=1847,
-    symbols=[
-        Symbol(qualified_name="pkg.AuthMiddleware", kind="function", score=0.78, provenance="lsp_resolved", distance=0),
-        Symbol(qualified_name="pkg.NewServer", kind="function", score=0.54, provenance="lsp_resolved", distance=1),
-    ],
-    edges=[
-        Edge(source="pkg.NewServer", target="pkg.AuthMiddleware", edge_type="calls"),
+output = encode_generic({
+    "employees": [
+        {"id": 1, "name": "Alice", "department": "Engineering", "salary": 95000},
+        {"id": 2, "name": "Bob", "department": "Sales", "salary": 72000},
     ],
-)
-
-output = encode(p)
+})
 ```
 
 Output:
 ```
-GCF tool=context_for_task budget=5000 tokens=1847 symbols=2 edges=1
-## targets
-@0 fn pkg.AuthMiddleware 0.78 lsp_resolved
-## related
-@1 fn pkg.NewServer 0.54 lsp_resolved
-## edges [1]
-@0<@1 calls
+## employees [2]{id,name,department,salary}
+1|Alice|Engineering|95000
+2|Bob|Sales|72000
 ```
 
 ## Decode
@@ -119,6 +107,35 @@ out2 = encode_with_session(payload2, sess)  # reused symbols as "@N  # previousl
 
 By the 5th call in a session: 92.7% token savings vs JSON.
 
+## Streaming Encode
+
+Write GCF output incrementally as symbols and edges arrive. Zero buffering, O(1) memory per row:
+
+```python
+from gcf import StreamEncoder, Symbol, Edge
+
+enc = StreamEncoder(sys.stdout, "context_for_task", token_budget=5000)
+
+enc.write_symbol(Symbol(qualified_name="pkg.Auth", kind="function", score=0.95, provenance="lsp", distance=0))
+enc.write_symbol(Symbol(qualified_name="pkg.Server", kind="function", score=0.60, provenance="lsp", distance=1))
+enc.write_edge(Edge(source="pkg.Server", target="pkg.Auth", edge_type="calls"))
+enc.close()  # emits ## _summary trailer
+```
+
+Output:
+```
+GCF tool=context_for_task budget=5000
+## targets
+@0 fn pkg.Auth 0.95 lsp
+## related
+@1 fn pkg.Server 0.60 lsp
+## edges [?]
+@0<@1 calls
+## _summary symbols=2 edges=1 sections=targets:1,related:1,edges:1
+```
+
+The writer is any object with a `write(s: str)` method. Thread-safe. Standard `decode()` handles streaming output with no changes.
+
 ## Delta Encoding
 
 When the consumer already has a prior context pack, send only what changed:

{gcf_python-0.3.1.dist-info → gcf_python-0.5.0.dist-info}/RECORD

@@ -1,14 +1,17 @@
-gcf/__init__.py,sha256=epW-CoQh3RBrnF3-lWSiPS-DI4HUlRRe8q9JeLPRxwE,1541
+gcf/__init__.py,sha256=ihKGNwTM0cYDi3t2o4PJjcRjaRW-rE6LYZKPg_KAldE,1738
 gcf/cli.py,sha256=2hSTBqiYcn1_EgGXuO65MHiEGh0C4DRMvspTd2zUaso,4258
 gcf/constants.py,sha256=cmZ8YJSOB0im_eyfN8v4UvrLpBC6Fuf4cfcKZGbutxY,638
 gcf/decode.py,sha256=48G6XmBilGYl0-c_Xy7MX0iTwOxGW9bn3wC4UyNApc4,5465
+gcf/decode_generic.py,sha256=NWRhOPWHypCDw6BbnDq78J7QA7EmfuBhvRyDKxBLwXg,7292
 gcf/delta.py,sha256=xU0ujtSq1iF7yU8yk_WNQKh8iove-WUV_nKSuvW1XVk,1656
 gcf/encode.py,sha256=Oljb1r5b7SHmng1XYvcvuJMpaRfJVJ81VOyocPf_kAs,2915
 gcf/generic.py,sha256=Nf1Ii0pYS0dgZ4o2ghl-3Qhnms-kCKY2D3q-3-oGpe8,5454
 gcf/session.py,sha256=4_ARRL06Tg2CI8D2eyi0V5nFphFAFMfOKBXYnAbI6Nk,4690
+gcf/stream.py,sha256=DBzZrb9t5ldVEsvcRX90SY59Fm0PM4aPt4XBIrJ1Gro,5186
+gcf/stream_generic.py,sha256=udKtvj-YJXl_eG_KzMZ3g9_EBMS3BTcAmAnsczPJ90M,3826
 gcf/types.py,sha256=AWm-LQoSqLHAYtEjcAxWQZqJ4JXqNreLUKO2mJFgNMA,1465
-gcf_python-0.3.1.dist-info/METADATA,sha256=xQxbAQEk1jr_c_GTsFQM1zyCmuz_MmgcbSxc1hNcKz0,8127
-gcf_python-0.3.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
-gcf_python-0.3.1.dist-info/entry_points.txt,sha256=aFT6gqlkh8iGfM8cblE-LUMxHH08_v71IIoZtDdRIVA,37
-gcf_python-0.3.1.dist-info/licenses/LICENSE,sha256=txSvg3E4LugiB7MOOTci6WKd6wMOrOJTvaITeFJ2SgU,1074
-gcf_python-0.3.1.dist-info/RECORD,,
+gcf_python-0.5.0.dist-info/METADATA,sha256=M399fPoVeSL4AvQqm7HaR92rkVCilg5MC1jzqWHwSSg,8667
+gcf_python-0.5.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+gcf_python-0.5.0.dist-info/entry_points.txt,sha256=aFT6gqlkh8iGfM8cblE-LUMxHH08_v71IIoZtDdRIVA,37
+gcf_python-0.5.0.dist-info/licenses/LICENSE,sha256=txSvg3E4LugiB7MOOTci6WKd6wMOrOJTvaITeFJ2SgU,1074
+gcf_python-0.5.0.dist-info/RECORD,,