bpp-format 0.3.0__py3-none-any.whl

bpp/__init__.py

@@ -0,0 +1,36 @@
+"""bpp - a token-efficient text format for feeding structured data to LLMs.
+
+    import bpp
+    text = bpp.dumps({"users": [{"id": 1, "name": "Ayşe"}]})   # -> .bpp text
+    data = bpp.loads(text)                                      # -> Python objects
+    data = bpp.load("config.yaml")          # any supported file (.json .yaml .csv .md .bpp)
+    bpp.dump(data, "config.bpp")            # format from the extension
+"""
+
+from pathlib import Path
+
+from .decoder import decode
+from .encoder import encode
+from .lexer import BppError
+
+__version__ = "0.3.0"
+__all__ = ["encode", "decode", "dumps", "loads", "load", "dump", "BppError", "__version__"]
+
+dumps = encode
+loads = decode
+
+
+def load(path):
+    """Read a .json/.yaml/.csv/.md/.bpp file into Python objects."""
+    from .formats import detect, loads as _loads
+
+    return _loads(Path(path).read_text(encoding="utf-8"), detect(path))
+
+
+def dump(data, path, **options):
+    """Write data to a file; the extension picks the format (.bpp by default)."""
+    from .formats import detect, dumps as _dumps
+
+    fmt = detect(path) if Path(path).suffix else "bpp"
+    with open(path, "w", encoding="utf-8", newline="\n") as f:  # newline=: Python 3.9
+        f.write(_dumps(data, fmt, **options))

bpp/__main__.py

@@ -0,0 +1,7 @@
+"""`python -m bpp ...` works even when the `bpp` script is not on PATH."""
+
+import sys
+
+from .cli import main
+
+sys.exit(main())

bpp/cli.py

@@ -0,0 +1,239 @@
+"""Command line interface: bpp encode | decode | stats."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+from . import __version__
+from .formats import FORMATS, detect, dumps, loads
+
+
+def _read(path: str) -> str:
+    if path == "-":
+        return sys.stdin.read()
+    return Path(path).read_text(encoding="utf-8")
+
+
+def _write(path: str | None, text: str):
+    if not path or path == "-":
+        sys.stdout.write(text)
+    else:
+        with open(path, "w", encoding="utf-8", newline="\n") as f:
+            f.write(text)
+
+
+def _load(path: str, fmt: str | None):
+    fmt = fmt or (detect(path) if path != "-" else None)
+    if not fmt:
+        raise ValueError("reading stdin needs --from")
+    return loads(_read(path), fmt), fmt
+
+
+def cmd_encode(a) -> int:
+    data, fmt = _load(a.input, a.from_)
+    primer = {"none": False, "short": True, "long": "long"}[a.primer]
+    # CSV column order is part of the data, so never move a column there.
+    _write(a.output, dumps(data, "bpp", primer=primer, refs=not a.no_refs,
+                           keep_order=a.keep_order or fmt == "csv"))
+    return 0
+
+
+def cmd_decode(a) -> int:
+    data, _ = _load(a.input, a.from_ or "bpp")
+    to = a.to or (detect(a.output) if a.output and a.output != "-" else "json")
+    kw = {"indent": None if a.indent < 0 else a.indent} if to == "json" else {}
+    _write(a.output, dumps(data, to, **kw))
+    return 0
+
+
+# ---------------------------------------------------------------------- stats
+
+def _toon_bridge() -> list[str] | None:
+    """Command that runs the reference TOON encoder, if available."""
+    node = shutil.which("node")
+    if not node:
+        return None
+    candidates = [os.environ.get("BPP_TOON_BRIDGE"),
+                  Path.cwd() / "bench/toon/toon.mjs",
+                  Path(__file__).resolve().parents[2] / "bench/toon/toon.mjs"]
+    for c in candidates:
+        if c and Path(c).exists() and (Path(c).parent / "node_modules").exists():
+            return [node, str(c), "encode"]
+    return None
+
+
+def renderings(data, keep_order: bool = False, markdown: bool = False) -> dict[str, str]:
+    """The same data in every format we compare."""
+    import json
+
+    out = {
+        "JSON (indent 2)": dumps(data, "json"),
+        "JSON (minified)": dumps(data, "json", indent=None),
+        "YAML": dumps(data, "yaml"),
+    }
+    try:
+        out["CSV"] = dumps(data, "csv")
+    except ValueError:
+        pass
+    if markdown:
+        out["Markdown"] = dumps(data, "md")
+    bridge = _toon_bridge()
+    if bridge:
+        p = subprocess.run(bridge, input=json.dumps(data, ensure_ascii=False),
+                           capture_output=True, text=True, encoding="utf-8")
+        if p.returncode == 0:
+            out["TOON"] = p.stdout
+    out["bpp"] = dumps(data, "bpp", keep_order=keep_order)
+    out["bpp + primer"] = dumps(data, "bpp", primer=True, keep_order=keep_order)
+    return out
+
+
+def cmd_stats(a) -> int:
+    from .tokens import available_counters
+
+    data, fmt = _load(a.input, a.from_)
+    counters = available_counters()
+    rows = [(name, text, {c: fn(text) for c, fn in counters.items()})
+            for name, text in renderings(data, keep_order=fmt == "csv",
+                                             markdown=fmt == "md").items()]
+    base = rows[0][2]
+    names = list(counters)
+    head = ["format", "chars"] + names + [f"vs JSON ({n})" for n in names]
+    table = [head]
+    for name, text, cnt in rows:
+        table.append([name, str(len(text))] + [str(cnt[n]) for n in names]
+                     + [f"{100 * (cnt[n] - base[n]) / base[n]:+.1f}%" for n in names])
+    if a.markdown:
+        print("| " + " | ".join(head) + " |")
+        print("|" + "---|" * len(head))
+        for r in table[1:]:
+            print("| " + " | ".join(r) + " |")
+    else:
+        w = [max(len(r[i]) for r in table) for i in range(len(head))]
+        for r in table:
+            print("  ".join(c.ljust(w[i]) if i == 0 else c.rjust(w[i]) for i, c in enumerate(r)))
+    if "estimate" in counters:
+        print("\nnote: 'estimate' is approximate; pip install tiktoken for real token counts.",
+              file=sys.stderr)
+    elif "anthropic" not in counters:
+        print("\nnote: o200k/claude2 are proxies; set ANTHROPIC_API_KEY for real Claude counts.",
+              file=sys.stderr)
+    return 0
+
+
+# ------------------------------------------------------------ one-step mode
+
+def _savings(src: str, out: str) -> str:
+    try:
+        from .tokens import available_counters
+
+        name, fn = next(iter(available_counters().items()))
+    except Exception:
+        return ""
+    a, b = fn(src), fn(out)
+    if not a:
+        return ""
+    if name == "estimate":
+        return (f" (~{a} -> ~{b} tokens, {100 * (b - a) / a:+.0f}%, rough estimate; "
+                "pip install tiktoken for exact counts)")
+    return f" ({name}: {a} -> {b} tokens, {100 * (b - a) / a:+.0f}%)"
+
+
+def cmd_auto(a) -> int:
+    """`bpp FILE`: .bpp files are decoded, everything else is encoded."""
+    src = Path(a.input)
+    if src.suffix.lower() == ".bpp":
+        to = a.to or (detect(a.output) if a.output and a.output != "-" else "json")
+        out = a.output or str(src.with_suffix("." + {"yaml": "yaml", "md": "md",
+                                                        "csv": "csv"}.get(to, "json")))
+        if out != "-" and Path(out).exists() and not a.force:
+            raise ValueError(f"{out} exists; use -o to pick another name or --force to overwrite")
+        text = dumps(loads(_read(a.input), "bpp"), to)
+        _write(out, text)
+    else:
+        fmt = detect(src)
+        raw = _read(a.input)
+        out = a.output or str(src.with_suffix(".bpp"))
+        text = dumps(loads(raw, fmt), "bpp", primer=a.primer, keep_order=fmt == "csv")
+        _write(out, text)
+        if out != "-":
+            print(f"{src} -> {out}{_savings(raw, text)}", file=sys.stderr)
+    return 0
+
+
+def _auto_main(argv: list[str]) -> int:
+    p = argparse.ArgumentParser(
+        prog="bpp", description="bpp FILE: encode json/yaml/csv/md to FILE.bpp, or decode "
+        "FILE.bpp to json. Subcommands: encode, decode, stats (bpp <cmd> -h).")
+    p.add_argument("input")
+    p.add_argument("-o", "--output", help="output file, '-' for stdout (default: next to input)")
+    p.add_argument("--to", choices=["json", "yaml", "csv", "md"], help="decode target format")
+    p.add_argument("--primer", action="store_true", help="add a one-line format explanation")
+    p.add_argument("-f", "--force", action="store_true", help="overwrite when decoding")
+    a = p.parse_args(argv)
+    try:
+        return cmd_auto(a)
+    except (ValueError, OSError) as ex:
+        print(f"bpp: error: {ex}", file=sys.stderr)
+        return 1
+
+
+def _utf8_console():
+    # Windows consoles default to a legacy code page; Turkish text would crash print().
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8")
+        except (AttributeError, ValueError):
+            pass
+
+
+def main(argv: list[str] | None = None) -> int:
+    _utf8_console()
+    argv = sys.argv[1:] if argv is None else argv
+    if argv and argv[0] not in ("encode", "decode", "stats", "-h", "--help", "--version"):
+        return _auto_main(argv)
+    p = argparse.ArgumentParser(prog="bpp", description="Token-efficient data format for LLMs. "
+                                "Shortcut: bpp FILE (encode, or decode if FILE is .bpp).")
+    p.add_argument("--version", action="version", version=f"bpp {__version__}")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    e = sub.add_parser("encode", help="convert json/yaml/csv/md to .bpp")
+    e.add_argument("input", help="input file ('-' for stdin with --from)")
+    e.add_argument("-o", "--output", help="output .bpp file (default stdout)")
+    e.add_argument("--from", dest="from_", choices=[f for f in FORMATS if f != "bpp"])
+    e.add_argument("--primer", choices=["none", "short", "long"], default="none",
+                   help="prepend a format explanation for LLMs that have not seen bpp")
+    e.add_argument("--no-refs", action="store_true", help="disable the &n/*n dictionary")
+    e.add_argument("--keep-order", action="store_true",
+                   help="never reorder keys (slightly larger output)")
+    e.set_defaults(fn=cmd_encode)
+
+    d = sub.add_parser("decode", help="convert .bpp to json/yaml/csv/md")
+    d.add_argument("input", help="input .bpp file ('-' for stdin)")
+    d.add_argument("-o", "--output", help="output file (default stdout)")
+    d.add_argument("--to", choices=["json", "yaml", "csv", "md"],
+                   help="output format (default: from -o extension, else json)")
+    d.add_argument("--indent", type=int, default=2, help="JSON indent; -1 = minified")
+    d.set_defaults(fn=cmd_decode, from_=None)
+
+    s = sub.add_parser("stats", help="compare token counts across formats")
+    s.add_argument("input")
+    s.add_argument("--from", dest="from_", choices=list(FORMATS))
+    s.add_argument("--markdown", action="store_true", help="print a Markdown table")
+    s.set_defaults(fn=cmd_stats)
+
+    a = p.parse_args(argv)
+    try:
+        return a.fn(a)
+    except (ValueError, OSError) as ex:
+        print(f"bpp: error: {ex}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bpp/decoder.py

@@ -0,0 +1,402 @@
+""".bpp text -> JSON data model (SPEC §1-§6)."""
+
+from __future__ import annotations
+
+import re
+
+from .lexer import BppError, Cursor
+
+_COUNT_RE = re.compile(r"\[(\d+)\]")
+
+
+def _header(text: str):
+    """`[N]` or `[N]{...}` at the start of text -> (N, rest) or None."""
+    m = _COUNT_RE.match(text)
+    if not m or (m.end() < len(text) and text[m.end()] != "{"):
+        return None
+    return int(m.group(1)), text[m.end():]
+
+
+class _Spec:
+    """Parsed column spec of a table or row table (see encoder._Spec)."""
+
+    def __init__(self, cols, delim, child, sub):
+        self.cols, self.delim, self.child, self.sub = cols, delim, child, sub
+        self.order = [c[0] for c in cols]
+        self.smode = {c[0]: c[3] for c in cols}
+        self.optional = {c[0] for c in cols if c[1]}
+        self.keyed = {c[0] for c in cols if c[2]}
+        self.positional = [p for p in self.order[:-1] if p not in self.keyed]
+
+
+def _set_path(obj: dict, path: tuple, v, line):
+    for k in path[:-1]:
+        nxt = obj.setdefault(k, {})
+        if not isinstance(nxt, dict):
+            raise BppError(f"column {'.'.join(path)!r} conflicts with {k!r}", line)
+        obj = nxt
+    obj[path[-1]] = v
+
+
+class _Line:
+    __slots__ = ("depth", "text", "no")
+
+    def __init__(self, depth, text, no):
+        self.depth, self.text, self.no = depth, text, no
+
+
+def decode(text: str):
+    """Decode .bpp text into Python/JSON values."""
+    raw = text.split("\n")
+    lines: list[_Line] = []
+    version = 0
+    for no, ln in enumerate(raw, 1):
+        if ln.endswith("\r"):
+            ln = ln[:-1]
+        stripped = ln.lstrip(" ")
+        if not stripped or stripped.startswith("#"):
+            continue
+        if not version:
+            if stripped not in ("bpp1", "bpp2", "bpp3"):
+                raise BppError("missing 'bpp3' header", no)
+            version = int(stripped[3])
+            continue
+        lines.append(_Line(len(ln) - len(stripped), stripped, no))
+    if not version:
+        raise BppError("missing 'bpp3' header")
+    return _Dec(lines, version).document()
+
+
+class _Dec:
+    def __init__(self, lines: list[_Line], version: int = 2):
+        self.L = lines
+        self.version = version
+        self.i = 0
+        self.refs: list = []
+
+    # -- helpers ---------------------------------------------------------------
+    def cur(self, ln: _Line, start: int = 0) -> Cursor:
+        c = Cursor(ln.text, self.refs, ln.no)
+        c.i = start
+        return c
+
+    def depth_at(self, i: int) -> int:
+        return self.L[i].depth if i < len(self.L) else -1
+
+    # -- document ----------------------------------------------------------------
+    def document(self):
+        while self.i < len(self.L) and self.L[self.i].text.startswith("&"):
+            ln = self.L[self.i]
+            m = re.match(r"&(\d+) ", ln.text)
+            if not m or ln.depth or int(m.group(1)) != len(self.refs):
+                raise BppError("bad dictionary definition", ln.no)
+            c = self.cur(ln, m.end())
+            v = c.value()
+            if not isinstance(v, str) or not c.eof():
+                raise BppError("dictionary value must be a string", ln.no)
+            self.refs.append(v)
+            self.i += 1
+        if self.i >= len(self.L):
+            raise BppError("empty document")
+        first = self.L[self.i]
+        if first.depth:
+            raise BppError("unexpected indentation", first.no)
+        if self.i == len(self.L) - 1:
+            v = self._root_inline(first)
+            if v is not _NO:
+                self.i += 1
+                return v
+        if first.text.startswith("["):
+            v = self.keyless(first, 0)
+        else:
+            v = self.object(0)
+        if self.i < len(self.L):
+            raise BppError("unexpected content", self.L[self.i].no)
+        return v
+
+    def _root_inline(self, ln: _Line):
+        c = self.cur(ln)
+        ch = c.peek()
+        try:
+            if ch == '"':
+                v = c.jstring()
+            elif ch in "[{" or ch == "*":
+                v = c.value(stops=" ")
+            else:
+                v = c.token(" ")
+                if isinstance(v, str):
+                    return _NO
+        except BppError:
+            return _NO
+        return v if c.eof() else _NO
+
+    # -- objects -----------------------------------------------------------------
+    def object(self, d: int) -> dict:
+        obj: dict = {}
+        while self.i < len(self.L):
+            ln = self.L[self.i]
+            if ln.depth < d:
+                break
+            if ln.depth > d:
+                raise BppError("unexpected indentation", ln.no)
+            if ln.text.startswith("- ") or ln.text == "-":
+                raise BppError("list item outside a list", ln.no)
+            self.i += 1
+            k, v = self.entry(ln, 0, d)
+            obj[k] = v
+        return obj
+
+    def entry(self, ln: _Line, start: int, d: int):
+        """Parse `key ...` beginning at column `start`; `d` is its logical depth."""
+        c = self.cur(ln, start)
+        key = c.key()
+        rest = ln.text[c.i:]
+        if rest == "":
+            if self.depth_at(self.i) != d + 1:
+                raise BppError(f"key {key!r} has no value", ln.no)
+            return key, self.object(d + 1)
+        if rest[0] == " ":
+            c.i += 1
+            v = c.value()
+            if not c.eof():
+                raise BppError("trailing characters", ln.no)
+            return key, v
+        if rest[0] == "[":
+            return key, self.block(rest, ln, d)
+        raise BppError(f"expected space after key {key!r}", ln.no)
+
+    # -- arrays ------------------------------------------------------------------
+    def keyless(self, ln: _Line, d: int):
+        """A line at depth d beginning with '[': header `[N]...` or inline list."""
+        h = _header(ln.text)
+        if h and (h[1] or self._items_follow(d)):
+            self.i += 1
+            return self.block(ln.text, ln, d)
+        c = self.cur(ln)
+        v = c.inline_list()
+        if not c.eof():
+            raise BppError("trailing characters", ln.no)
+        self.i += 1
+        return v
+
+    def _items_follow(self, d: int) -> bool:
+        j = self.i + 1
+        return j < len(self.L) and self.L[j].depth == d and (
+            self.L[j].text.startswith("- ") or self.L[j].text == "-")
+
+    def block(self, head: str, ln: _Line, d: int) -> list:
+        h = _header(head)
+        if not h:
+            raise BppError("bad array header", ln.no)
+        n, rest = h
+        if not rest:
+            return self.items(n, d, ln)
+        c = self.cur(_Line(0, rest, ln.no))
+        spec = self._spec(c)
+        if not c.eof():
+            raise BppError("bad array header", ln.no)
+        if spec.child is None and not spec.optional and (len(spec.cols) == 1 or spec.delim == ","):
+            return self.table(n, spec, d, ln)
+        return self.outline(n, spec, d, ln)
+
+    def _spec(self, c: Cursor) -> _Spec:
+        dotted = self.version >= 3
+        c.expect("{")
+        cols = []
+        delim = None
+        while True:
+            path = c.path(dotted)
+            opt = keyed = False
+            if c.peek() == "?":
+                opt = True
+                c.i += 1
+                # bpp2+: `x?` = positional, '-' when absent; `x?=` = written as x=v.
+                # bpp1 only had the x=v form, spelled `x?`.
+                if self.version == 1:
+                    keyed = True
+                elif c.peek() == "=":
+                    keyed = True
+                    c.i += 1
+            smode = False
+            if c.s.startswith(":str", c.i):
+                smode = True
+                c.i += 4
+            cols.append((path, opt, keyed, smode))
+            sep = c.peek()
+            if sep == "}":
+                c.i += 1
+                break
+            if delim is None:
+                delim = sep
+            if sep != delim or sep not in ", ":
+                c.err("bad column list")
+            c.i += 1
+        child = sub = None
+        if c.peek() == ">":
+            c.i += 1
+            child = c.segment() if dotted else c.key()
+            if dotted and c.peek() == "{":
+                sub = self._spec(c)
+        return _Spec(cols, delim or ",", child, sub)
+
+    def table(self, n: int, spec: _Spec, d: int, ln: _Line) -> list:
+        rows = []
+        for _ in range(n):
+            if self.i >= len(self.L) or self.L[self.i].depth != d:
+                raise BppError(f"expected {n} table rows", ln.no)
+            r = self.L[self.i]
+            self.i += 1
+            c = self.cur(r)
+            obj: dict = {}
+            for j, p in enumerate(spec.order):
+                _set_path(obj, p, c.token(",", spec.smode[p]), r.no)
+                if j < len(spec.order) - 1:
+                    c.expect(",")
+            if not c.eof():
+                raise BppError("too many cells", r.no)
+            rows.append(obj)
+        return rows
+
+    def outline(self, n: int, spec: _Spec, d: int, ln: _Line) -> list:
+        dotted = self.version >= 3
+
+        def check(sp):
+            if sp.order[-1] in sp.optional:
+                raise BppError("last column must be required", ln.no)
+            if sp.sub is not None:
+                check(sp.sub)
+        check(spec)
+
+        def row(r: _Line, sp: _Spec):
+            c = self.cur(r)
+            got = {}
+            child_empty = False
+            for p in sp.positional:
+                if p in sp.optional and c.s.startswith("- ", c.i):
+                    c.i += 2  # '-' = this optional column is absent
+                    continue
+                got[p] = self._cell(c, sp.smode[p])
+                c.expect(" ")
+            while True:
+                save = c.i
+                if c.peek() == '"' or c.peek() not in "[{*":
+                    try:
+                        p = c.path(dotted)
+                    except BppError:
+                        c.i = save
+                        break
+                    if c.peek() == "=":
+                        if sp.child is not None and p == (sp.child,) and not child_empty:
+                            c.i += 1
+                            if c.value(" ") != []:
+                                raise BppError("child key may only be [] inline", r.no)
+                            child_empty = True
+                            c.expect(" ")
+                            continue
+                        if p in sp.keyed and p not in got:
+                            c.i += 1
+                            got[p] = self._cell(c, sp.smode[p])
+                            c.expect(" ")
+                            continue
+                c.i = save
+                break
+            last = sp.order[-1]
+            got[last] = c.value(strmode=sp.smode[last])
+            if not c.eof():
+                raise BppError("trailing characters", r.no)
+            obj: dict = {}
+            for p in sp.order:
+                if p in got:
+                    _set_path(obj, p, got[p], r.no)
+            if child_empty:
+                obj[sp.child] = []
+            return obj
+
+        def rows_at(depth: int, count, sp: _Spec) -> list:
+            out = []
+            while self.i < len(self.L) and (count is None or len(out) < count):
+                r = self.L[self.i]
+                if r.depth < depth:
+                    break
+                if r.depth > depth:
+                    raise BppError("unexpected indentation", r.no)
+                self.i += 1
+                obj = row(r, sp)
+                if sp.child is not None and self.depth_at(self.i) == depth + 1:
+                    if sp.child in obj:
+                        raise BppError("child rows after child=[]", r.no)
+                    obj[sp.child] = rows_at(depth + 1, None, sp.sub or sp)
+                out.append(obj)
+            if count is not None and len(out) != count:
+                raise BppError(f"expected {count} rows", ln.no)
+            return out
+
+        return rows_at(d, n, spec)
+
+    @staticmethod
+    def _cell(c: Cursor, smode: bool):
+        if c.peek() == "[":
+            return c.inline_list(smode)
+        return c.value(" ", smode)
+
+    def items(self, n: int, d: int, ln: _Line) -> list:
+        out = []
+        for _ in range(n):
+            if self.i >= len(self.L) or self.L[self.i].depth != d:
+                raise BppError(f"expected {n} list items", ln.no)
+            it = self.L[self.i]
+            if not it.text.startswith("- "):
+                raise BppError("expected '- ' list item", it.no)
+            self.i += 1
+            out.append(self.item(it, d))
+        return out
+
+    def item(self, it: _Line, d: int):
+        body = it.text[2:]
+        sub = _Line(d + 1, body, it.no)
+        if body.startswith("["):
+            h = _header(body)
+            if h and (h[1] or (
+                    self.i < len(self.L) and self.L[self.i].depth == d + 1
+                    and self.L[self.i].text.startswith("- "))):
+                return self.block(body, sub, d + 1)
+            c = self.cur(sub)
+            v = c.inline_list()
+            if not c.eof():
+                raise BppError("trailing characters", it.no)
+            return v
+        if body == "{}":
+            return {}
+        c = self.cur(sub)
+        try:
+            key = c.key()
+            rest = body[c.i:]
+        except BppError:
+            key, rest = None, None
+        if key is not None:
+            is_entry = (rest.startswith(" ") or rest.startswith("[")
+                        or (rest == "" and self.depth_at(self.i) == d + 2))
+            if is_entry:
+                k, v = self.entry(sub, 0, d + 1)
+                obj = {k: v}
+                more = self.object(d + 1)
+                for kk, vv in more.items():
+                    if kk in obj:
+                        raise BppError(f"duplicate key {kk!r}", it.no)
+                    obj[kk] = vv
+                return obj
+        c = self.cur(sub)
+        if c.peek() == '"':
+            v = c.jstring()
+        else:
+            v = c.value()
+        if not c.eof():
+            raise BppError("trailing characters", it.no)
+        return v
+
+
+class _Sentinel:
+    pass
+
+
+_NO = _Sentinel()