bpp-format 0.3.0__py3-none-any.whl

bpp/lexer.py

@@ -0,0 +1,241 @@
+"""Scalar, key and inline-value syntax shared by the encoder and decoder (SPEC §2-§4)."""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+
+NUM_RE = re.compile(r"-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?")
+SPECIAL = {"null": None, "true": True, "false": False,
+           "NaN": math.nan, "Infinity": math.inf, "-Infinity": -math.inf}
+_CTRL = re.compile(r"[\x00-\x1f\x7f]")
+BARE_KEY_RE = re.compile(r"[^\s\"\[\]{}=,:?>\x00-\x1f\x7f]+")
+BARE_SEG_RE = re.compile(r"[^\s\"\[\]{}=,:?>.\x00-\x1f\x7f]+")  # a column-path segment
+_KV_START = re.compile(r"(?:[^\s\"\[\]{}=,:?>]+|\"(?:[^\"\\]|\\.)*\")=")
+_DECODER = json.JSONDecoder()
+
+
+class BppError(ValueError):
+    """Raised for malformed .bpp input."""
+
+    def __init__(self, msg: str, line: int | None = None):
+        super().__init__(f"line {line}: {msg}" if line else msg)
+        self.line = line
+
+
+class Ref:
+    """Placeholder for a dictionary reference (`*n`) produced by the encoder."""
+
+    __slots__ = ("idx",)
+
+    def __init__(self, idx: int):
+        self.idx = idx
+
+
+def is_scalar(v) -> bool:
+    return v is None or isinstance(v, (str, bool, int, float, Ref))
+
+
+def jstr(s: str) -> str:
+    return json.dumps(s, ensure_ascii=False)
+
+
+# ---------------------------------------------------------------- formatting
+
+def fmt_number(v) -> str:
+    if isinstance(v, int):
+        return str(v)
+    if math.isnan(v):
+        return "NaN"
+    if math.isinf(v):
+        return "Infinity" if v > 0 else "-Infinity"
+    return repr(v)
+
+
+def needs_quote(s: str, ctx: str = "value", strmode: bool = False) -> bool:
+    """Minimum-quoting rule (SPEC §2.1). ctx: value|cell|list|pos|item|last."""
+    if (not s or s != s.strip() or _CTRL.search(s) or s[0] in '"[{*&#'
+            or s in SPECIAL):
+        return True
+    if not strmode and NUM_RE.fullmatch(s):
+        return True
+    if ctx == "cell":
+        return "," in s
+    if ctx == "list":
+        return "," in s or "]" in s
+    if ctx == "pos":
+        return " " in s or s == "-"
+    if ctx == "item":
+        return " " in s or "[" in s or "{" in s
+    if ctx == "last":
+        return _KV_START.match(s) is not None
+    return False
+
+
+def fmt_scalar(v, ctx: str = "value", strmode: bool = False) -> str:
+    if isinstance(v, Ref):
+        return f"*{v.idx}"
+    if v is None:
+        return "null"
+    if v is True:
+        return "true"
+    if v is False:
+        return "false"
+    if isinstance(v, (int, float)):
+        return fmt_number(v)
+    return jstr(v) if needs_quote(v, ctx, strmode) else v
+
+
+def fmt_key(k: str) -> str:
+    if BARE_KEY_RE.fullmatch(k) and k[0] not in "-#&*":
+        return k
+    return jstr(k)
+
+
+def fmt_seg(k: str) -> str:
+    """A key inside a table header, where '.' separates path segments."""
+    if BARE_SEG_RE.fullmatch(k) and k[0] not in "-#&*":
+        return k
+    return jstr(k)
+
+
+def fmt_inline(v, ctx: str = "value", strmode: bool = False) -> str | None:
+    """Single-line form of v, or None if v needs a block."""
+    if is_scalar(v):
+        return fmt_scalar(v, ctx, strmode)
+    if isinstance(v, dict):
+        return "{}" if not v else None
+    if isinstance(v, list):
+        if all(is_scalar(x) for x in v):
+            return "[" + ",".join(fmt_scalar(x, "list", strmode) for x in v) + "]"
+        return None
+    raise TypeError(f"unsupported type {type(v).__name__}")
+
+
+# ------------------------------------------------------------------- parsing
+
+def parse_bare(tok: str, strmode: bool = False):
+    """Interpret an unquoted token (no surrounding whitespace)."""
+    if strmode:
+        return None if tok == "null" else tok
+    if tok in SPECIAL:
+        return SPECIAL[tok]
+    if NUM_RE.fullmatch(tok):
+        if "." in tok or "e" in tok or "E" in tok:
+            return float(tok)
+        return int(tok)
+    return tok
+
+
+class Cursor:
+    """Scans one line of text; `refs` resolves `*n`."""
+
+    def __init__(self, text: str, refs: list, line: int | None):
+        self.s, self.i, self.refs, self.line = text, 0, refs, line
+
+    def err(self, msg):
+        raise BppError(msg, self.line)
+
+    def eof(self):
+        return self.i >= len(self.s)
+
+    def peek(self):
+        return self.s[self.i] if self.i < len(self.s) else ""
+
+    def expect(self, ch):
+        if self.peek() != ch:
+            self.err(f"expected {ch!r} at column {self.i + 1}")
+        self.i += 1
+
+    def jstring(self) -> str:
+        try:
+            val, end = _DECODER.raw_decode(self.s, self.i)
+        except json.JSONDecodeError as e:
+            self.err(f"bad quoted string: {e.msg}")
+        if not isinstance(val, str):
+            self.err("expected string")
+        self.i = end
+        return val
+
+    def key(self) -> str:
+        if self.peek() == '"':
+            return self.jstring()
+        m = BARE_KEY_RE.match(self.s, self.i)
+        if not m or self.s[self.i] in "-#&*":
+            self.err("expected key")
+        self.i = m.end()
+        return m.group()
+
+    def segment(self) -> str:
+        if self.peek() == '"':
+            return self.jstring()
+        m = BARE_SEG_RE.match(self.s, self.i)
+        if not m or self.s[self.i] in "-#&*":
+            self.err("expected key")
+        self.i = m.end()
+        return m.group()
+
+    def path(self, dotted: bool) -> tuple:
+        """A column path `a.b.c` (bpp3); in bpp1/2 headers a column is one key."""
+        if not dotted:
+            return (self.key(),)
+        out = [self.segment()]
+        while self.peek() == ".":
+            self.i += 1
+            out.append(self.segment())
+        return tuple(out)
+
+    def resolve(self, tok: str, strmode: bool):
+        if tok.startswith("*") and tok[1:].isdigit():
+            idx = int(tok[1:])
+            if idx >= len(self.refs):
+                self.err(f"undefined reference {tok}")
+            return self.refs[idx]
+        return parse_bare(tok, strmode)
+
+    def token(self, stops: str, strmode: bool = False):
+        """A scalar ending before any char in `stops` (or end of line)."""
+        if self.peek() == '"':
+            return self.jstring()
+        j = self.i
+        while j < len(self.s) and self.s[j] not in stops:
+            j += 1
+        tok = self.s[self.i:j]
+        if not tok:
+            self.err(f"empty value at column {self.i + 1}")
+        self.i = j
+        return self.resolve(tok, strmode)
+
+    def inline_list(self, strmode: bool = False) -> list:
+        self.expect("[")
+        out = []
+        if self.peek() == "]":
+            self.i += 1
+            return out
+        while True:
+            out.append(self.token(",]", strmode))
+            c = self.peek()
+            self.i += 1
+            if c == "]":
+                return out
+            if c != ",":
+                self.err("unterminated inline list")
+
+    def value(self, stops: str = "", strmode: bool = False):
+        """Inline value: scalar, [list], {} (for `key value` stops is empty -> rest of line)."""
+        c = self.peek()
+        if c == "[":
+            return self.inline_list(strmode)
+        if self.s.startswith("{}", self.i):
+            self.i += 2
+            return {}
+        if not stops:
+            if c == '"':
+                return self.jstring()
+            tok = self.s[self.i:]
+            if not tok:
+                self.err("missing value")
+            self.i = len(self.s)
+            return self.resolve(tok, strmode)
+        return self.token(stops, strmode)

bpp/markdown.py

@@ -0,0 +1,160 @@
+"""Markdown plans <-> tree (SPEC §7).
+
+Headings and nested lists become a tree of steps:
+
+    {"title": "...", "note": "...", "steps": [
+        {"title": "...", "status": "done", "note": "...", "steps": [...]}]}
+
+* A single level-1 heading becomes the document title.
+* `- [ ]`, `- [x]`, `- [/]`, `- [-]` set status todo / done / doing / cancelled
+  (also allowed right after the `#`s of a heading).
+* Paragraphs, code blocks and other text go to the `note` of the heading or
+  list item they belong to.
+
+The conversion keeps structure, not formatting: `tree_to_md` writes a
+normalized Markdown document (top-level steps as `##`, deeper steps as `-`
+lists) which parses back to the same tree.
+"""
+
+from __future__ import annotations
+
+import re
+
+_HEADING = re.compile(r"(#{1,6})\s+(.*?)\s*#*\s*$")
+_ITEM = re.compile(r"( *)([-*+]|\d{1,9}[.)])(?: +(.*))?$")
+_BOX = re.compile(r"\[([ xX/-])\] +")
+_FENCE = re.compile(r"(```|~~~)")
+STATUS = {" ": "todo", "x": "done", "X": "done", "/": "doing", "-": "cancelled"}
+BOX = {"todo": " ", "done": "x", "doing": "/", "cancelled": "-"}
+
+
+def _node(text: str) -> dict:
+    m = _BOX.match(text)
+    node: dict = {"title": (text[m.end():] if m else text).strip()}
+    if m:
+        node["status"] = STATUS[m.group(1)]
+    return node
+
+
+def _add_note(node: dict, line: str):
+    node.setdefault("_note", []).append(line)
+
+
+def md_to_tree(text: str) -> dict:
+    root: dict = {"steps": []}
+    headings: list[tuple[int, dict]] = [(0, root)]
+    items: list[tuple[int, int, dict]] = []  # (indent, content column, node)
+    fence = None  # (marker, target node, strip column)
+    h1_nodes: list[dict] = []
+
+    for raw in text.replace("\r\n", "\n").split("\n"):
+        line = raw.expandtabs(4)
+        stripped = line.lstrip(" ")
+        indent = len(line) - len(stripped)
+
+        if fence:
+            marker, node, col = fence
+            _add_note(node, line[min(col, indent):] if stripped else "")
+            if stripped.startswith(marker):
+                fence = None
+            continue
+
+        if not stripped:
+            target = items[-1][2] if items else headings[-1][1]
+            if target.get("_note") and target["_note"][-1] != "":
+                _add_note(target, "")
+            continue
+
+        m = _HEADING.match(stripped) if indent < 4 else None
+        if m:
+            level = len(m.group(1))
+            node = _node(m.group(2))
+            if level == 1:
+                h1_nodes.append(node)
+            while headings[-1][0] >= level:
+                headings.pop()
+            headings[-1][1].setdefault("steps", []).append(node)
+            headings.append((level, node))
+            items = []
+            continue
+
+        m = _ITEM.match(line)
+        if m:
+            while items and items[-1][0] >= indent:
+                items.pop()
+            node = _node(m.group(3) or "")
+            parent = items[-1][2] if items else headings[-1][1]
+            parent.setdefault("steps", []).append(node)
+            items.append((indent, indent + len(m.group(2)) + 1, node))
+            continue
+
+        # text: goes to the innermost list item it is indented under
+        while items and indent < items[-1][1] and indent <= items[-1][0]:
+            items.pop()
+        if items:
+            node, col = items[-1][2], items[-1][1]
+        else:
+            node, col = headings[-1][1], 0
+        f = _FENCE.match(stripped)
+        if f:
+            fence = (f.group(1), node, col)
+        _add_note(node, line[min(col, indent):])
+
+    def finish(node: dict):
+        note = node.pop("_note", None)
+        while note and note[-1] == "":
+            note.pop()
+        if note:
+            node["note"] = "\n".join(note)
+        for child in node.get("steps", []):
+            finish(child)
+        return node
+
+    finish(root)
+    doc: dict = {}
+    top = root.get("steps", [])
+    if len(h1_nodes) == 1 and "status" not in h1_nodes[0]:
+        # The only '# ' heading is the document title.  Anything that came
+        # before it (text, list items, lower headings) stays in front.
+        t = h1_nodes[0]
+        doc["title"] = t["title"]
+        notes = [n for n in (root.get("note"), t.get("note")) if n]
+        if notes:
+            doc["note"] = "\n\n".join(notes)
+        doc["steps"] = [n for n in top if n is not t] + t.get("steps", [])
+        return doc
+    if root.get("note"):
+        doc["note"] = root["note"]
+    doc["steps"] = top
+    return doc
+
+
+def tree_to_md(doc: dict) -> str:
+    if not isinstance(doc, dict) or not isinstance(doc.get("steps", []), list):
+        raise ValueError("Markdown output needs a plan tree: {title?, note?, steps: [...]}")
+    out: list[str] = []
+    if doc.get("title") is not None:
+        out += [f"# {doc['title']}", ""]
+    if doc.get("note"):
+        out += [doc["note"], ""]
+    for node in doc.get("steps", []):
+        box = f"[{BOX[node['status']]}] " if node.get("status") in BOX else ""
+        out.append(f"## {box}{node.get('title', '')}")
+        out.append("")
+        if node.get("note"):
+            out += [node["note"], ""]
+        _items(node.get("steps", []), 0, out)
+        if node.get("steps"):
+            out.append("")
+    return "\n".join(out).rstrip("\n") + "\n"
+
+
+def _items(nodes, depth, out):
+    pad = "  " * depth
+    for node in nodes:
+        box = f"[{BOX[node['status']]}] " if node.get("status") in BOX else ""
+        out.append(f"{pad}- {box}{node.get('title', '')}".rstrip())
+        if node.get("note"):
+            for ln in str(node["note"]).split("\n"):
+                out.append(f"{pad}  {ln}" if ln else "")
+        _items(node.get("steps", []), depth + 1, out)

bpp/tokens.py

@@ -0,0 +1,185 @@
+"""Token counters used by `bpp stats` and the benchmarks.
+
+Three counters, all optional:
+
+* ``o200k``   - tiktoken ``o200k_base`` (modern OpenAI BPE; a rough proxy).
+* ``claude2`` - the legacy Claude tokenizer Anthropic published in the
+  ``@anthropic-ai/tokenizer`` npm package (Claude 2 era; a second proxy that
+  behaves more like Claude on non-English text).
+* ``anthropic`` - the real ``messages.count_tokens`` endpoint.  Used only when
+  the ``anthropic`` SDK is installed and ``ANTHROPIC_API_KEY`` is set.
+
+Neither proxy is exact for current Claude models; relative comparisons between
+formats are what matter here.
+
+tiktoken normally downloads its rank files from a blob store that some
+networks block.  If that fails we fetch the same ranks from the npm registry
+(``js-tiktoken`` ships them), verify the published SHA-256, and drop the file
+into tiktoken's own cache so later calls work offline.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import io
+import json
+import os
+import tarfile
+import tempfile
+import urllib.request
+from functools import lru_cache
+from pathlib import Path
+from typing import Callable
+
+O200K_URL = "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken"
+O200K_SHA256 = "446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2d"
+JS_TIKTOKEN_TGZ = "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz"
+CLAUDE_TGZ = "https://registry.npmjs.org/@anthropic-ai/tokenizer/-/tokenizer-0.0.4.tgz"
+
+ANTHROPIC_MODEL = os.environ.get("BPP_ANTHROPIC_MODEL", "claude-opus-5")
+
+
+def _cache_dir() -> Path:
+    d = os.environ.get("BPP_CACHE_DIR") or os.path.join(
+        os.path.expanduser("~"), ".cache", "bpp"
+    )
+    Path(d).mkdir(parents=True, exist_ok=True)
+    return Path(d)
+
+
+def _tiktoken_cache_dir() -> Path:
+    # Mirrors tiktoken.load.read_file_cached
+    if "TIKTOKEN_CACHE_DIR" in os.environ:
+        d = os.environ["TIKTOKEN_CACHE_DIR"]
+    elif "DATA_GYM_CACHE_DIR" in os.environ:
+        d = os.environ["DATA_GYM_CACHE_DIR"]
+    else:
+        d = os.path.join(tempfile.gettempdir(), "data-gym-cache")
+    Path(d).mkdir(parents=True, exist_ok=True)
+    return Path(d)
+
+
+def _fetch_tgz_member(url: str, member_suffix: str) -> bytes:
+    with urllib.request.urlopen(url, timeout=60) as r:
+        data = r.read()
+    with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tf:
+        for m in tf.getmembers():
+            if m.name.endswith(member_suffix):
+                f = tf.extractfile(m)
+                assert f is not None
+                return f.read()
+    raise FileNotFoundError(f"{member_suffix} not in {url}")
+
+
+def _parse_js_ranks(text: str) -> dict:
+    """Parse js-tiktoken's compressed rank format: '! <offset> <b64> <b64> ...'."""
+    if not text.lstrip().startswith("{"):
+        text = text[text.index("{") :].rstrip().rstrip(";")
+    return json.loads(text)
+
+
+def _expand_ranks(bpe_ranks: str) -> dict[bytes, int]:
+    ranks: dict[bytes, int] = {}
+    for line in bpe_ranks.split("\n"):
+        if not line:
+            continue
+        _, offset, *toks = line.split(" ")
+        for i, t in enumerate(toks):
+            ranks[base64.b64decode(t)] = int(offset) + i
+    return ranks
+
+
+def bootstrap_o200k() -> None:
+    """Populate tiktoken's cache with o200k_base fetched via npm."""
+    path = _tiktoken_cache_dir() / hashlib.sha1(O200K_URL.encode()).hexdigest()
+    if path.exists():
+        return
+    d = _parse_js_ranks(
+        _fetch_tgz_member(JS_TIKTOKEN_TGZ, "dist/ranks/o200k_base.js").decode()
+    )
+    ranks = _expand_ranks(d["bpe_ranks"])
+    body = "".join(
+        base64.b64encode(b).decode() + " " + str(r) + "\n"
+        for b, r in sorted(ranks.items(), key=lambda kv: kv[1])
+    ).encode()
+    if hashlib.sha256(body).hexdigest() != O200K_SHA256:
+        raise ValueError("o200k_base rebuilt from npm does not match tiktoken's hash")
+    path.write_bytes(body)
+
+
+@lru_cache(maxsize=None)
+def _o200k():
+    import tiktoken
+
+    try:
+        return tiktoken.get_encoding("o200k_base")
+    except Exception:
+        bootstrap_o200k()
+        return tiktoken.get_encoding("o200k_base")
+
+
+@lru_cache(maxsize=None)
+def _claude2():
+    import tiktoken
+
+    path = _cache_dir() / "claude.json"
+    if not path.exists():
+        path.write_bytes(_fetch_tgz_member(CLAUDE_TGZ, "package/claude.json"))
+    d = json.loads(path.read_text(encoding="utf-8"))
+    return tiktoken.Encoding(
+        name="claude2",
+        pat_str=d["pat_str"],
+        mergeable_ranks=_expand_ranks(d["bpe_ranks"]),
+        special_tokens=d["special_tokens"],
+    )
+
+
+@lru_cache(maxsize=None)
+def _anthropic_client():
+    import anthropic
+
+    return anthropic.Anthropic()
+
+
+@lru_cache(maxsize=4096)
+def _count_anthropic(text: str) -> int:
+    resp = _anthropic_client().messages.count_tokens(
+        model=ANTHROPIC_MODEL, messages=[{"role": "user", "content": text}]
+    )
+    return resp.input_tokens
+
+
+def _count_anthropic_net(text: str) -> int:
+    # count_tokens includes a fixed per-message overhead; subtract it so that
+    # the numbers are comparable to the local tokenizers.
+    return _count_anthropic(text) - _count_anthropic("x") + 1
+
+
+def available_counters() -> dict[str, Callable[[str], int]]:
+    """Return the counters that work in this environment, name -> fn(text)."""
+    out: dict[str, Callable[[str], int]] = {}
+    try:
+        enc = _o200k()
+        out["o200k"] = lambda s: len(enc.encode_ordinary(s))
+    except Exception:
+        pass
+    try:
+        enc2 = _claude2()
+        out["claude2"] = lambda s: len(enc2.encode_ordinary(s))
+    except Exception:
+        pass
+    if os.environ.get("ANTHROPIC_API_KEY"):
+        try:
+            _count_anthropic("x")
+            out["anthropic"] = _count_anthropic_net
+        except Exception:
+            pass
+    if not out:
+        # Last resort so `bpp stats` still prints something useful: the
+        # encoder's own estimator (ranks formats like the real tokenizers do;
+        # a plain chars/4 gets the direction wrong for Turkish text).
+        from .estimate import est_tokens
+
+        out["estimate"] = est_tokens
+    return out