flattype 1.11.0__py3-none-any.whl

flattype/_path.py

@@ -0,0 +1,298 @@
+"""Paths (``user.address.zip``, ``rows[2]{id,name}``) and building the decoded value from them."""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Callable, NoReturn, Union
+
+from ._syntax import FlatDecodeError, format_key, format_number, is_key_char, scan_string
+
+#: A key (str) or an array index (int; a float only for indexes too large to be valid).
+Segment = Union[str, int, float]
+#: ("value",) | ("inline", delimiter) | ("table", count, delimiter, fields)
+Terminal = tuple[Any, ...]
+
+VALUE: Terminal = ("value",)
+_DIGITS = re.compile(r"[0-9]+")
+
+#: The most nulls lenient decoding inserts for a skipped index range.
+MAX_GAP = 1000
+
+
+def _at(s: str, i: int) -> str:
+    return s[i] if i < len(s) else ""
+
+
+def _number(digits: str) -> int | float:
+    # Like JavaScript's Number(digits): exact up to 15 digits, a float (maybe inf) beyond.
+    return int(digits) if len(digits) <= 15 else float(digits)
+
+
+def show_number(n: int | float) -> str:
+    return str(n) if isinstance(n, int) else format_number(n)
+
+
+def parse_path(s: str, pos: int, line: int) -> tuple[list[Segment], Terminal, int]:
+    """Reads a path starting at ``pos``. Stops at the first character that can't continue a path (normally ``:``)."""
+    segments: list[Segment] = []
+
+    def fail(msg: str, at: int) -> NoReturn:
+        raise FlatDecodeError(msg, line, at + 1)
+
+    need_key = False
+    while True:
+        c = _at(s, pos)
+        if c == '"' or (c != "" and is_key_char(c)):
+            if segments and not need_key:
+                fail("Expected '.' or ':' before key", pos)
+            pos = _read_key(s, pos, segments, fail)
+            need_key = False
+            if _at(s, pos) == ".":
+                pos += 1
+                need_key = True
+            continue
+        if need_key:
+            fail("Expected a key after '.'", pos)
+        if c != "[":
+            break
+        close = s.find("]", pos)
+        if close < 0:
+            fail("Unclosed '['", pos)
+        inner = s[pos + 1 : close]
+        if inner in ("", "|", "\t"):
+            return segments, ("inline", inner or ","), close + 1
+        digits, mark = inner, ""
+        if inner[-1] in "|\t":
+            digits, mark = inner[:-1], inner[-1]
+        if not _DIGITS.fullmatch(digits):
+            fail(f"Invalid index '[{inner}]'", pos)
+        n = _number(digits)
+        pos = close + 1
+        if _at(s, pos) == "{":
+            delimiter = mark or ","
+            fields, end = _read_fields(s, pos + 1, delimiter, fail)
+            return segments, ("table", n, delimiter, fields), end
+        if mark != "":
+            fail("A delimiter mark is only allowed on inline arrays and tables", pos - 1)
+        segments.append(n)
+        if _at(s, pos) == ".":
+            pos += 1
+            need_key = True
+    return segments, VALUE, pos
+
+
+def _read_key(s: str, pos: int, out: list[Segment], fail: Callable[[str, int], NoReturn]) -> int:
+    if s[pos] == '"':
+        close = scan_string(s, pos)
+        if close < 0:
+            fail("Unterminated quoted key", pos)
+        try:
+            out.append(json.loads(s[pos : close + 1]))
+        except ValueError:
+            fail("Invalid quoted key", pos)
+        return close + 1
+    end = pos
+    n = len(s)
+    while end < n and is_key_char(s[end]):
+        end += 1
+    out.append(s[pos:end])
+    return end
+
+
+def _read_fields(s: str, pos: int, delimiter: str, fail: Callable[[str, int], NoReturn]) -> tuple[list[list[Segment]], int]:
+    """Reads ``id,user.name}`` (after the ``{``) into one key path per field."""
+    fields: list[list[Segment]] = []
+    while True:
+        while _at(s, pos) == " ":
+            pos += 1
+        field: list[Segment] = []
+        while True:
+            c = _at(s, pos)
+            if c != '"' and not (c != "" and is_key_char(c)):
+                fail("Expected a field name", pos)
+            pos = _read_key(s, pos, field, fail)
+            if _at(s, pos) != ".":
+                break
+            pos += 1
+        fields.append(field)
+        while _at(s, pos) == " ":
+            pos += 1
+        c = _at(s, pos)
+        if c == "}":
+            return fields, pos + 1
+        if c != delimiter:
+            fail(f"Expected '{chr(92) + 't' if delimiter == chr(9) else delimiter}' or '}}' in table header", pos)
+        pos += 1
+
+
+class Ref:
+    """Placeholder for a ``path = target`` line until the whole document has been read."""
+
+    __slots__ = ("segments", "target", "line")
+
+    def __init__(self, segments: list[Segment], target: list[Segment], line: int) -> None:
+        self.segments = segments
+        self.target = target
+        self.line = line
+
+
+def assign(holder: dict[str, Any], segments: list[Segment], value: Any, strict: bool, line: int) -> None:
+    """Writes ``value`` at ``segments`` under ``holder["root"]``, creating dicts and lists on the way."""
+    parent: Any = holder
+    key: Segment = "root"
+
+    def fail(msg: str) -> NoReturn:
+        raise FlatDecodeError(msg, line, 1)
+
+    def slot(create: Callable[[], Any], leaf: bool) -> Any:
+        if not isinstance(key, str):
+            arr: list[Any] = parent
+            if key < len(arr):
+                if leaf:
+                    if strict:
+                        fail(f"Duplicate index [{show_number(key)}]")
+                    arr[int(key)] = create()
+                return arr[int(key)]
+            if key > len(arr):
+                if strict:
+                    fail(f"Index [{show_number(key)}] skips [{len(arr)}]")
+                # Lenient mode fills small gaps with None, but `a[999999999]: 1` must not allocate a billion entries.
+                if key - len(arr) > MAX_GAP:
+                    fail(f"Index [{show_number(key)}] is too far past [{len(arr)}]")
+                arr.extend([None] * (int(key) - len(arr)))
+            v = create()
+            arr.append(v)
+            return v
+        obj: dict[str, Any] = parent
+        if key in obj:
+            if not leaf:
+                return obj[key]
+            if strict:
+                fail(f"Duplicate key '{key}'")
+        v = create()
+        obj[key] = v
+        return v
+
+    for seg in segments:
+        want_list = not isinstance(seg, str)
+        cur = slot(list if want_list else dict, False)
+        if not isinstance(cur, list if want_list else dict):
+            fail(f"Path conflict at {'[' + show_number(key) + ']' if not isinstance(key, str) else repr_key(key)}")
+        parent = cur
+        key = seg
+    slot(lambda: value, True)
+
+
+def check(holder: dict[str, Any], segments: list[Segment], strict: bool, line: int) -> None:
+    """Raises exactly the error :func:`assign` would, without changing anything, so a line that fails leaves no
+    trace (used when lenient decoding skips lines it can't read)."""
+
+    def fail(msg: str) -> NoReturn:
+        raise FlatDecodeError(msg, line, 1)
+
+    # What sits at `key` in `parent`; `fresh` means `parent` doesn't exist yet (it would be created empty).
+    parent: Any = holder
+    key: Segment = "root"
+    fresh = False
+
+    def lookup(leaf: bool) -> Any:
+        if not isinstance(key, str):
+            length = 0 if fresh else len(parent)
+            if key < length:
+                if leaf and strict:
+                    fail(f"Duplicate index [{show_number(key)}]")
+                return _MISSING if leaf else parent[int(key)]
+            if key > length:
+                if strict:
+                    fail(f"Index [{show_number(key)}] skips [{length}]")
+                if key - length > MAX_GAP:
+                    fail(f"Index [{show_number(key)}] is too far past [{length}]")
+            return _MISSING
+        if not fresh and key in parent:
+            if not leaf:
+                return parent[key]
+            if strict:
+                fail(f"Duplicate key '{key}'")
+        return _MISSING
+
+    for seg in segments:
+        want_list = not isinstance(seg, str)
+        cur = lookup(False)
+        if cur is _MISSING:
+            fresh = True
+        else:
+            if not isinstance(cur, list if want_list else dict):
+                fail(f"Path conflict at {'[' + show_number(key) + ']' if not isinstance(key, str) else repr_key(key)}")
+            parent = cur
+            fresh = False
+        key = seg
+    lookup(True)
+
+
+def repr_key(key: str) -> str:
+    return f"'{key}'"
+
+
+_MISSING = object()
+
+
+def _js_get(container: Any, seg: Segment) -> Any:
+    """``container[seg]`` if it is an own property in JavaScript terms (so ``[0]`` on a dict reads key "0")."""
+    if isinstance(container, dict):
+        k = seg if isinstance(seg, str) else show_number(seg)
+        return container.get(k, _MISSING)
+    if isinstance(container, list):
+        if isinstance(seg, str):
+            return len(container) if seg == "length" else _MISSING
+        return container[int(seg)] if 0 <= seg < len(container) else _MISSING
+    return _MISSING
+
+
+def _show(segments: list[Segment]) -> str:
+    if not segments:
+        return "~"
+    out = ""
+    for i, s in enumerate(segments):
+        if not isinstance(s, str):
+            out += f"[{show_number(s)}]"
+        else:
+            out += format_key(s) if i == 0 else "." + format_key(s)
+    return out
+
+
+def resolve_refs(holder: dict[str, Any], refs: list[Ref], strict: bool) -> None:
+    """Replaces each placeholder with the dict or list its target points to, in document order.
+
+    A target may pass through a reference resolved before it, but not through one still waiting.
+    """
+    for ref in refs:
+        # Lenient decoding may have overwritten the placeholder (or a parent) with a later line; that line wins.
+        parent: Any = holder.get("root", _MISSING)
+        last = len(ref.segments) - 1
+        for seg in ref.segments[:last]:
+            if not isinstance(parent, (dict, list)):
+                break
+            parent = _js_get(parent, seg)
+        if not isinstance(parent, (dict, list)) or _js_get(parent, ref.segments[last]) is not ref:
+            continue
+
+        cur: Any = holder.get("root", _MISSING)
+        for seg in ref.target:
+            cur = _js_get(cur, seg) if isinstance(cur, (dict, list)) else _MISSING
+        value = cur
+        if not isinstance(cur, (dict, list)):
+            if isinstance(cur, Ref):
+                why = "is a reference that comes later"
+            elif cur is _MISSING:
+                why = "doesn't exist"
+            else:
+                why = "isn't an object or array"
+            if strict:
+                raise FlatDecodeError(f"Reference target '{_show(ref.target)}' {why}", ref.line, 1)
+            value = None
+        k = ref.segments[last]
+        if isinstance(parent, list):
+            parent[int(k)] = value
+        else:
+            parent[k if isinstance(k, str) else show_number(k)] = value

flattype/_syntax.py

@@ -0,0 +1,171 @@
+"""Lexical rules shared by the encoder and decoder. Every rule matches @flattype/core exactly."""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+from typing import Any
+
+
+class FlatDecodeError(ValueError):
+    """Raised by :func:`decode` on malformed input, with the 1-based ``line`` and ``column`` of the problem."""
+
+    def __init__(self, message: str, line: int, column: int) -> None:
+        super().__init__(f"{message} (line {line}, column {column})")
+        self.line = line
+        self.column = column
+
+
+#: Exactly the JSON number grammar (ASCII digits only). Anything else bare is a string.
+NUMBER = re.compile(r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?")
+_BARE_KEY = re.compile(r"[A-Za-z0-9_$-]+")
+_CONTROL = re.compile(r"[\x00-\x1f]")
+
+#: The characters JavaScript's String#trim removes (WhiteSpace and LineTerminator).
+JS_SPACE = "\t\n\v\f\r              

  　"
+
+#: CPython refuses (or takes quadratic time) to convert very long digit strings to int.
+_MAX_INT_DIGITS = 4300
+
+
+def js_trim(s: str) -> str:
+    return s.strip(JS_SPACE)
+
+
+def is_key_char(c: str) -> bool:
+    """Characters allowed in an unquoted key: A-Z a-z 0-9 _ $ -"""
+    return ("a" <= c <= "z") or ("A" <= c <= "Z") or ("0" <= c <= "9") or c in "_$-"
+
+
+_ESCAPES = {'"': '\\"', "\\": "\\\\", "\b": "\\b", "\f": "\\f", "\n": "\\n", "\r": "\\r", "\t": "\\t"}
+_NEEDS_ESCAPE = re.compile('["\\\\\x00-\x1f\ud800-\udfff]')
+
+
+def _escape(m: re.Match[str]) -> str:
+    c = m.group()
+    return _ESCAPES.get(c) or f"\\u{ord(c):04x}"
+
+
+def quote(s: str) -> str:
+    """A JSON string literal, byte for byte what JavaScript's JSON.stringify writes (lone surrogates escaped)."""
+    return '"' + _NEEDS_ESCAPE.sub(_escape, s) + '"'
+
+
+def format_key(key: str) -> str:
+    return key if _BARE_KEY.fullmatch(key) else quote(key)
+
+
+def join_key(path: str, key: str) -> str:
+    return format_key(key) if path == "" else path + "." + format_key(key)
+
+
+def format_string(s: str, delimiter: str | None = None) -> str:
+    """Writes a string bare when it can't be mistaken for anything else, quoted (JSON style) otherwise."""
+    if (
+        s == ""
+        or s[0] == '"'
+        or s != js_trim(s)
+        or s in ("null", "true", "false", "{}", "[]")
+        or NUMBER.fullmatch(s)
+        or _CONTROL.search(s)
+        or (delimiter is not None and delimiter in s)
+    ):
+        return quote(s)
+    return s
+
+
+def format_number(x: float) -> str:
+    """Formats a number the way JavaScript does (``1e+21``, ``1e-7``, ``100``), so both libraries write the same text.
+
+    Integers are written exactly. Floats use the shortest round-trip digits, like ``repr``, laid out by the
+    ECMAScript Number::toString rules. ``NaN`` and infinities become ``null``, as in JSON.
+    """
+    if isinstance(x, int):
+        return str(x)
+    if not math.isfinite(x):
+        return "null"
+    if x == 0:
+        return "0"
+    r = repr(abs(x))
+    mant, _, exp = r.partition("e")
+    whole, _, frac = mant.partition(".")
+    digits = whole + frac
+    n = len(whole) + (int(exp) if exp else 0)  # x = 0.digits * 10**n
+    stripped = digits.lstrip("0")
+    n -= len(digits) - len(stripped)
+    digits = stripped.rstrip("0")
+    k = len(digits)
+    if k <= n <= 21:
+        out = digits + "0" * (n - k)
+    elif 0 < n <= 21:
+        out = digits[:n] + "." + digits[n:]
+    elif -6 < n <= 0:
+        out = "0." + "0" * -n + digits
+    else:
+        e = n - 1
+        sign = "+" if e >= 0 else "-"
+        out = (digits if k == 1 else digits[0] + "." + digits[1:]) + "e" + sign + str(abs(e))
+    return "-" + out if x < 0 else out
+
+
+def format_primitive(v: Any, delimiter: str | None = None) -> str:
+    if isinstance(v, str):
+        return format_string(v, delimiter)
+    if v is True:
+        return "true"
+    if v is False:
+        return "false"
+    if v is None:
+        return "null"
+    if isinstance(v, int):
+        return str(int(v))
+    return format_number(float(v))
+
+
+def to_number(tok: str) -> int | float:
+    """A token that matched NUMBER: an int when it has no fraction or exponent, otherwise a float."""
+    if "." in tok or "e" in tok or "E" in tok or len(tok) > _MAX_INT_DIGITS:
+        return float(tok)
+    return int(tok)
+
+
+#: Returned by parse_scalar for a malformed quoted token.
+INVALID = object()
+
+
+def parse_scalar(tok: str) -> Any:
+    """Parses one trimmed bare or quoted token. Returns ``INVALID`` when a quoted token is malformed."""
+    if tok == "null":
+        return None
+    if tok == "true":
+        return True
+    if tok == "false":
+        return False
+    if tok == "{}":
+        return {}
+    if tok == "[]":
+        return []
+    if tok[:1] == '"':
+        try:
+            v = json.loads(tok)
+        except ValueError:
+            return INVALID
+        return v if isinstance(v, str) else INVALID
+    return to_number(tok) if NUMBER.fullmatch(tok) else tok
+
+
+_QUOTE_OR_BACKSLASH = re.compile(r'["\\]')
+
+
+def scan_string(s: str, start: int) -> int:
+    """Index of the closing quote of the JSON string starting at ``start``, or -1."""
+    i = start + 1
+    while True:
+        m = _QUOTE_OR_BACKSLASH.search(s, i)
+        if m is None:
+            return -1
+        j = m.start()
+        if s[j] == '"':
+            return j
+        i = j + 2  # skip the escaped character

flattype/ai.py

@@ -0,0 +1,154 @@
+"""Flat Notation for LLM prompts and replies: the Python side of ``@flattype/ai``.
+
+from flattype.ai import to_prompt, from_model_output
+
+content = f"{to_prompt(orders, name='orders')}\\n\\nWhich customer spent the most?"
+rows = from_model_output(reply_text)
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Callable, Protocol
+
+from ._decode import decode
+from ._encode import encode
+
+__all__ = ["LEGEND", "claude_tool_result", "count_tokens", "from_model_output", "output_instructions", "to_prompt", "tool_result"]
+
+#: One line that teaches a model the format. It is a ``#`` comment, so a block that includes it is still valid.
+LEGEND = (
+    "# Flat Notation: one `path: value` per line. `a.b` is a nested key, `a[0]` an array index, `a[]: x, y` a list. "
+    "`t[N]{f,g}:` is a table: N rows follow, one per line, values in field order. Quoted values are JSON strings."
+)
+
+
+def _attr(s: str) -> str:
+    return s.replace("&", "&amp;").replace('"', "&quot;").replace("<", "&lt;")
+
+
+def to_prompt(value: Any, *, name: str | None = None, legend: bool = True, tag: bool = True, **encode_options: Any) -> str:
+    """Formats data for an LLM prompt: Flat Notation inside a ``<data>`` tag, with a one-line legend.
+
+    :param name: adds ``name="..."`` to the tag, so a prompt can refer to several data blocks.
+    :param legend: include the format legend. Turn it off once a system prompt explains the format.
+    :param tag: wrap in ``<data format="flat-notation">...</data>``.
+    """
+    body = (LEGEND + "\n" if legend else "") + encode(value, **encode_options)
+    if not tag:
+        return body
+    return f'<data format="flat-notation"{f" name={chr(34)}{_attr(name)}{chr(34)}" if name else ""}>\n{body}\n</data>'
+
+
+def output_instructions(example: Any = None) -> str:
+    """Instructions that ask a model to answer in Flat Notation, with an example of the shape you expect."""
+    lines = [
+        "Reply with the data only, in Flat Notation, inside <data> and </data> tags.",
+        LEGEND[2:],
+        "Write a list of objects with the same keys as a table. Quote a string value if it contains a comma "
+        "or could be read as a number, true, false or null.",
+    ]
+    if example is not None:
+        lines += ["Example of the expected shape:", to_prompt(example, legend=False)]
+    return "\n".join(lines)
+
+
+_WORD = re.compile(r"[\w-]", re.ASCII)
+_JSON_LANG = re.compile(r"json", re.I)
+
+
+def _extract(text: str) -> tuple[str, bool]:
+    """The last ``<data>...</data>`` block, else the last fenced block, else the whole text. Linear scans only."""
+    lower = text.lower()
+    close = lower.rfind("</data>")
+    if close >= 0:
+        opening = lower.rfind("<data", 0, close + 5)
+        # `<data` must be the tag itself, not e.g. `<database>`.
+        while opening >= 0 and _WORD.match(lower[opening + 5 : opening + 6] or " "):
+            opening = lower.rfind("<data", 0, opening + 4) if opening > 0 else -1
+        gt = text.find(">", opening) if opening >= 0 else -1
+        if 0 <= gt < close:
+            return text[gt + 1 : close], False
+    fences: list[int] = []
+    i = text.find("```")
+    while i >= 0:
+        fences.append(i)
+        i = text.find("```", i + 3)
+    if len(fences) >= 2:
+        last = len(fences) - 2 if len(fences) % 2 == 0 else len(fences) - 3
+        start, end = fences[last], fences[last + 1]
+        nl = text.find("\n", start)
+        if 0 <= nl < end:
+            lang = text[start + 3 : nl].strip()
+            return text[nl + 1 : end], bool(_JSON_LANG.match(lang))
+    return text, False
+
+
+def from_model_output(text: str, model: Callable[[Any], Any] | None = None, *, many: bool = False) -> Any:
+    """Reads data out of an LLM reply, tolerating what models actually send: ``<data>`` tags or code fences around
+    it, prose before or after, small format slips, or plain JSON instead.
+
+    :param model: validates the result, e.g. a Pydantic model class (``model_validate`` is used) or any callable.
+    :param many: the data is a list; ``model`` is applied to each item.
+    """
+    body, is_json = _extract(text)
+    trimmed = body.strip()
+    value: Any = None
+    parsed = False
+    if is_json or trimmed[:1] in ("[", "{"):
+        try:
+            value = json.loads(trimmed)
+            parsed = True
+        except ValueError:
+            pass
+    if not parsed:
+        # Lenient, and lines that aren't Flat Notation (prose like "Here are the results:") are dropped.
+        value = decode(body, strict=False, skip_invalid_lines=True)
+    if model is None:
+        return value
+    validate = getattr(model, "model_validate", model)
+    return [validate(v) for v in value] if many else validate(value)
+
+
+class _Counter(Protocol):
+    def count_tokens(self, *, model: str, messages: list[dict[str, str]]) -> Any: ...
+
+
+class _Client(Protocol):
+    @property
+    def messages(self) -> _Counter: ...
+
+
+def count_tokens(client: _Client, value: Any, *, model: str = "claude-opus-5-5", **prompt_options: Any) -> dict[str, Any]:
+    """Tokens for ``value`` as minified JSON and as :func:`to_prompt` text, counted by Claude's token counter.
+
+    ``client`` is an ``anthropic.Anthropic()``. Counting is free. Returns ``{"model", "json", "flat", "saved"}``.
+    """
+
+    def count(content: str) -> int:
+        return int(client.messages.count_tokens(model=model, messages=[{"role": "user", "content": content}]).input_tokens)
+
+    # "." is one token; what's left is the message framing both counts share.
+    base = max(0, count(".") - 1)
+    j = count(json.dumps(value, separators=(",", ":"), ensure_ascii=False)) - base
+    f = count(to_prompt(value, **prompt_options)) - base
+    return {"model": model, "json": j, "flat": f, "saved": 0.0 if j == 0 else 1 - f / j}
+
+
+def tool_result(value: Any, *, is_error: bool = False, structured: bool = False, **prompt_options: Any) -> dict[str, Any]:
+    """An MCP tool result whose text is the data as Flat Notation (compatible with the ``mcp`` SDK's ``CallToolResult``)."""
+    result: dict[str, Any] = {"content": [{"type": "text", "text": to_prompt(value, **prompt_options)}]}
+    if structured and isinstance(value, dict):
+        result["structuredContent"] = json.loads(json.dumps(value))
+    if is_error:
+        result["isError"] = True
+    return result
+
+
+def claude_tool_result(tool_use_id: str, value: Any, *, is_error: bool = False, **prompt_options: Any) -> dict[str, Any]:
+    """A Claude API ``tool_result`` block whose content is the data as Flat Notation."""
+    block: dict[str, Any] = {"type": "tool_result", "tool_use_id": tool_use_id, "content": to_prompt(value, **prompt_options)}
+    if is_error:
+        block["is_error"] = True
+    return block