ltem-core 0.1.0__py3-none-any.whl

ltem/__init__.py

@@ -0,0 +1,51 @@
+"""LTEM (LLM-friendly Table-Enhanced Markdown).
+
+Reference Python implementation of the LTEM specification, version 1.0:
+Markdown pipe tables with merged cells and multi-row headers.
+
+    >>> import ltem
+    >>> result = ltem.parse_table('''
+    ... | Region | Revenue | <<  |
+    ... | ^^     | Q1      | Q2  |
+    ... |--------|---------|-----|
+    ... | Asia   | 100     | 120 |
+    ... '''.strip())
+    >>> [(c.text, c.rowspan, c.colspan) for c in result.table.cells][:2]
+    [('Region', 2, 1), ('Revenue', 1, 2)]
+"""
+
+from ._syntax import (ends_table_body, indentation, is_delimiter_row, is_upper_header_row, split_row,
+                      split_row_spans, starts_block)
+from .fromhtml import HtmlTable, from_html, html_to_ltem
+from .html import plain_inline, render_html
+from .model import Cell, CellSource, Problem, Table
+from .parser import ParseResult, find_tables, parse_table
+from .writer import write, write_with_sources
+
+__version__ = "0.1.0"
+SPEC_VERSION = "1.0"
+
+__all__ = [
+    "Cell",
+    "CellSource",
+    "HtmlTable",
+    "ParseResult",
+    "Problem",
+    "SPEC_VERSION",
+    "Table",
+    "find_tables",
+    "from_html",
+    "ends_table_body",
+    "html_to_ltem",
+    "indentation",
+    "is_delimiter_row",
+    "is_upper_header_row",
+    "parse_table",
+    "plain_inline",
+    "render_html",
+    "split_row",
+    "split_row_spans",
+    "starts_block",
+    "write",
+    "write_with_sources",
+]

ltem/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+import sys
+
+sys.exit(main())

ltem/_grid.py

@@ -0,0 +1,151 @@
+"""Span resolution (spec Section 3): grid rows, compact rows, markers and recovery."""
+
+from __future__ import annotations
+
+import copy
+from dataclasses import dataclass
+
+from ._syntax import COLSPAN_MARKER, ROWSPAN_MARKER, split_attribute
+from .model import Problem
+
+
+@dataclass
+class _Origin:
+    row: int
+    col: int
+    rowspan: int
+    colspan: int
+    text: str
+    fixed_cols: bool  # colspan came from an attribute: markers may not extend it
+    fixed_rows: bool  # rowspan came from an attribute
+    source: int | None = None  # index of the cell in its source row; None if added by padding
+
+
+class GridBuilder:
+    """Places the cells of a table row by row, as spec Section 3 describes."""
+
+    def __init__(self, cols: int, header_rows: int, rows: int) -> None:
+        self.cols, self.header_rows, self.rows = cols, header_rows, rows
+        self.owner: list[list[_Origin | None]] = [[None] * cols for _ in range(rows)]
+        self.origins: list[_Origin] = []
+        self.problems: list[Problem] = []
+
+    # -- helpers ---------------------------------------------------------------
+
+    def _report(self, rule: str, row: int, col: int | None, message: str) -> None:
+        self.problems.append(Problem(rule, row, col, message))
+
+    def _free(self, r0: int, r1: int, c0: int, c1: int) -> bool:
+        return all(self.owner[r][c] is None
+                   for r in range(r0, min(r1, self.rows)) for c in range(c0, min(c1, self.cols)))
+
+    def _cover(self, origin: _Origin, r0: int, r1: int, c0: int, c1: int) -> None:
+        for r in range(r0, min(r1, self.rows)):
+            for c in range(c0, min(c1, self.cols)):
+                self.owner[r][c] = origin
+
+    # -- placing (spec 3.3, 3.4) -------------------------------------------------
+
+    def _extend_left(self, r: int, c: int) -> bool:
+        left = self.owner[r][c - 1] if c > 0 else None
+        if left is None or left.row != r or left.col + left.colspan != c or left.fixed_cols:
+            self._report("V13", r, c, "'<<' has no cell to extend")
+            return False
+        if not self._free(r + 1, left.row + left.rowspan, c, c + 1):
+            self._report("V4", r, c, "'<<' would make two cells overlap")
+            return False
+        left.colspan += 1
+        self._cover(left, left.row, left.row + left.rowspan, c, c + 1)
+        return True
+
+    def _extend_up(self, r: int, c: int) -> bool:
+        up = self.owner[r - 1][c] if r > 0 else None
+        if up is None or up.col != c or up.row + up.rowspan != r or up.fixed_rows:
+            self._report("V13", r, c, "'^^' has no cell to extend")
+            return False
+        if up.row < self.header_rows <= r:
+            self._report("V7", r, c, "'^^' would extend a header cell into the body")
+            return False
+        if not self._free(r, r + 1, c + 1, up.col + up.colspan):
+            self._report("V4", r, c, "'^^' would make two cells overlap")
+            return False
+        up.rowspan += 1
+        self._cover(up, r, r + 1, up.col, up.col + up.colspan)
+        return True
+
+    def _place(self, r: int, c: int, text: str, source: int | None = None) -> None:
+        """Place a cell at the position (r, c), which is not covered."""
+        if text == COLSPAN_MARKER and self._extend_left(r, c):
+            return
+        if text == ROWSPAN_MARKER and self._extend_up(r, c):
+            return
+        colspan = rowspan = 1
+        fixed_cols = fixed_rows = False
+        if text not in (COLSPAN_MARKER, ROWSPAN_MARKER):  # an unusable marker stays literal text
+            text, attr = split_attribute(text)
+            if attr:
+                for message in attr.problems:
+                    self._report("V1", r, c, message)
+                colspan, fixed_cols = attr.colspan or 1, attr.colspan is not None
+                rowspan, fixed_rows = attr.rowspan or 1, attr.rowspan is not None
+        if c + colspan > self.cols:
+            self._report("V2", r, c, f"colspan {colspan} reaches past the last column")
+            colspan = self.cols - c
+        if r + rowspan > self.rows:
+            self._report("V3", r, c, f"rowspan {rowspan} reaches past the last row")
+            rowspan = self.rows - r
+        if r < self.header_rows < r + rowspan:
+            self._report("V7", r, c, "a header cell spans into the body")
+            rowspan = self.header_rows - r
+        # V4: cut to the largest rectangle that overlaps nothing, columns first, then rows.
+        k = 1
+        while k < colspan and self.owner[r][c + k] is None:
+            k += 1
+        j = 1
+        while j < rowspan and self._free(r + j, r + j + 1, c, c + k):
+            j += 1
+        if (k, j) != (colspan, rowspan):
+            self._report("V4", r, c, f"span cut from {colspan}x{rowspan} to {k}x{j} to avoid an overlap")
+            colspan, rowspan = k, j
+        origin = _Origin(r, c, rowspan, colspan, text, fixed_cols, fixed_rows, source)
+        self.origins.append(origin)
+        self._cover(origin, r, r + rowspan, c, c + colspan)
+
+    # -- rows (spec 3.2) ---------------------------------------------------------
+
+    def _compact_fits(self, r: int, texts: list[str]) -> bool:
+        trial = copy.deepcopy(self)
+        used = 0
+        for c in range(self.cols):
+            if trial.owner[r][c] is None:
+                if used == len(texts):
+                    return False
+                trial._place(r, c, texts[used])
+                used += 1
+        return all(t == "" for t in texts[used:])
+
+    def add_row(self, r: int, texts: list[str]) -> str:
+        """Place row r. Returns "grid", "compact" or "padded" (the row did not fit: V14)."""
+        n = len(texts)
+        if n != self.cols and self._compact_fits(r, texts):
+            remaining = iter(enumerate(texts))
+            for c in range(self.cols):
+                if self.owner[r][c] is None:
+                    index, text = next(remaining)
+                    self._place(r, c, text, index)
+            return "compact"
+        form = "grid"
+        if n != self.cols:
+            self._report("V14", r, None, f"row has {n} cells; it fits neither as a grid row "
+                                         f"({self.cols} cells) nor as a compact row")
+            texts = (texts + [""] * self.cols)[: self.cols]
+            form = "padded"
+        for c, text in enumerate(texts):
+            covering = self.owner[r][c]
+            if covering is not None:
+                if text not in ("", COLSPAN_MARKER, ROWSPAN_MARKER):
+                    rule = "V5" if covering.row < r else "V6"
+                    self._report(rule, r, c, f"a covered position holds text {text!r}; it is dropped")
+                continue
+            self._place(r, c, text, c if c < n else None)
+        return form

ltem/_syntax.py

@@ -0,0 +1,204 @@
+"""Line-level syntax: rows, delimiter rows, cell kinds and span attributes (spec Section 2)."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+DELIMITER_CELL = re.compile(r":?-+:?")
+# A span attribute ends the cell and is the whole cell or follows a space or tab (spec 2.6).
+SPAN_ATTRIBUTE = re.compile(r"(?:^|(?<=[ \t]))\{([cr]:[0-9]+(?:,[cr]:[0-9]+)*)\}$")
+# Reserved table-identifier line (spec Section 9).
+IDENTIFIER_ROW = re.compile(r"^ {0,3}\|[ \t]*#[ \t]+[A-Za-z_][A-Za-z0-9_]*[ \t]*\|[ \t]*$")
+PIPE_LED = re.compile(r"^ {0,3}\|")
+LINE_BREAK = re.compile(r"\\\\")
+
+COLSPAN_MARKER = "<<"
+ROWSPAN_MARKER = "^^"
+
+
+def split_row_spans(line: str) -> list[tuple[str, int, int]]:
+    """Split a row into cells, as GFM does (spec 2.1), keeping where each cell is.
+
+    Each cell is (text, start, end). `text` is trimmed of spaces and tabs, and a pipe
+    preceded by a backslash becomes a literal pipe. `start` and `end` index `line` and
+    cover the cell's source without the spaces and tabs around it; for an empty cell
+    both are the end of its space between the pipes.
+    """
+    b, e = 0, len(line)
+    while b < e and line[b] in " \t":
+        b += 1
+    while e > b and line[e - 1] in " \t":
+        e -= 1
+    if b < e and line[b] == "|":
+        b += 1
+    if e > b and line[e - 1] == "|" and not (e - 2 >= b and line[e - 2] == "\\"):
+        e -= 1
+    cells: list[tuple[str, int, int]] = []
+    cur: list[str] = []
+    cell_start = i = b
+    while i < e:
+        ch = line[i]
+        if ch == "\\" and i + 1 < e and line[i + 1] == "|":
+            cur.append("|")
+            i += 2
+            continue
+        if ch == "|":
+            cells.append(_cell_span(line, cell_start, i, cur))
+            cur = []
+            cell_start = i + 1
+        else:
+            cur.append(ch)
+        i += 1
+    cells.append(_cell_span(line, cell_start, e, cur))
+    return cells
+
+
+def _cell_span(line: str, start: int, end: int, chars: list[str]) -> tuple[str, int, int]:
+    while start < end and line[start] in " \t":
+        start += 1
+    while end > start and line[end - 1] in " \t":
+        end -= 1
+    return "".join(chars).strip(" \t"), start, end
+
+
+def split_row(line: str) -> list[str]:
+    """Split a row into trimmed cell texts, as GFM does (spec 2.1).
+
+    A pipe preceded by a backslash does not split cells and becomes a literal pipe.
+    """
+    return [text for text, _, _ in split_row_spans(line)]
+
+
+def indentation(line: str) -> int:
+    """Width of the leading spaces and tabs, with tab stops every 4 columns."""
+    width = 0
+    for ch in line:
+        if ch == " ":
+            width += 1
+        elif ch == "\t":
+            width += 4 - width % 4
+        else:
+            break
+    return width
+
+
+_LIST_DASH = re.compile(r"^ {0,3}-[ \t]")
+
+
+def is_delimiter_row(line: str) -> bool:
+    """A GFM delimiter row (spec 2.2).
+
+    It has at least one pipe, is indented less than 4 spaces, and does not begin a list
+    item: a line such as "- | -" is a list item, as in GFM.
+    """
+    if "|" not in line or indentation(line) >= 4 or _LIST_DASH.match(line):
+        return False
+    return all(DELIMITER_CELL.fullmatch(c) for c in split_row(line))
+
+
+def is_upper_header_row(line: str) -> bool:
+    """Whether a line can be a header row above the last one (spec 2.3, 9).
+
+    It begins with a pipe, is not a reserved identifier line and does not look like a
+    delimiter row.
+    """
+    return bool(PIPE_LED.match(line)) and not IDENTIFIER_ROW.match(line) and not is_delimiter_row(line)
+
+
+# Lines that start another block and so end a table body (spec 2.4).
+_BODY_END = re.compile(
+    r"^ {0,3}(?:"
+    r">"                                  # block quote
+    r"|#{1,6}(?:[ \t]|$)"                # ATX heading
+    r"|`{3,}|~{3,}"                       # fenced code
+    r"|([-*_])(?:[ \t]*\1){2,}[ \t]*$"     # thematic break
+    r"|(?:[-+*]|[0-9]{1,9}[.)])(?:[ \t]|$)"  # list item
+    r")"
+)
+
+
+_SETEXT_UNDERLINE = re.compile(r"^ {0,3}(?:=+|-+)[ \t]*$")
+
+
+def is_setext_underline(line: str) -> bool:
+    """A line of "=" or of "-" characters, which underlines a paragraph line as a heading."""
+    return bool(_SETEXT_UNDERLINE.match(line))
+
+
+def starts_block(line: str) -> bool:
+    """Whether a line begins a block quote, ATX heading, fenced code, thematic break or list item."""
+    return bool(_BODY_END.match(line))
+
+
+def ends_table_body(line: str) -> bool:
+    """Whether a table ends before this line (spec 2.4): blank, indented code, or another block."""
+    return not line.strip(" \t") or indentation(line) >= 4 or starts_block(line)
+
+
+def alignment(cell: str) -> str | None:
+    left, right = cell.startswith(":"), cell.endswith(":")
+    if left and right:
+        return "center"
+    if right:
+        return "right"
+    if left:
+        return "left"
+    return None
+
+
+@dataclass
+class Attribute:
+    colspan: int | None
+    rowspan: int | None
+    problems: list[str]
+
+
+def split_attribute(text: str) -> tuple[str, Attribute | None]:
+    """Remove a trailing span attribute from a content cell (spec 2.6).
+
+    Returns the remaining text and the attribute, or (text, None) if there is none.
+    """
+    m = SPAN_ATTRIBUTE.search(text)
+    if not m:
+        return text, None
+    colspan = rowspan = None
+    problems = []
+    for spec in m.group(1).split(","):
+        key, value = spec.split(":")
+        n = int(value)
+        if n < 1:
+            problems.append(f"span value must be a positive integer, got {value}")
+            n = 1
+        if key == "c":
+            if colspan is not None:
+                problems.append("'c' appears more than once")
+                continue
+            colspan = n
+        else:
+            if rowspan is not None:
+                problems.append("'r' appears more than once")
+                continue
+            rowspan = n
+    return text[: m.start()].rstrip(" \t"), Attribute(colspan, rowspan, problems)
+
+
+# The LTEM escapes of spec 4.3, as written in cell source.
+_ESCAPED_MARKER = re.compile(r"^\\(<<|\^\^)$")
+_ESCAPED_ATTRIBUTE = re.compile(r"(?:^|(?<=[ \t]))\\(\{[cr]:[0-9]+(?:,[cr]:[0-9]+)*\})$")
+
+
+def unescape_ltem(line: str) -> str:
+    """Remove the backslash of an LTEM escape (spec 4.3), as GFM does for \\|."""
+    m = _ESCAPED_MARKER.match(line)
+    if m:
+        return m.group(1)
+    return _ESCAPED_ATTRIBUTE.sub(r"\1", line)
+
+
+def cell_lines(text: str) -> str:
+    """Turn cell source into model text (spec Section 7).
+
+    Split at each \\\\, trim each line and remove LTEM escapes.
+    """
+    return "\n".join(unescape_ltem(part.strip(" \t")) for part in LINE_BREAK.split(text))

ltem/cli.py

@@ -0,0 +1,118 @@
+"""Command-line interface.
+
+    ltem parse    [FILE]   table models of all tables, as JSON
+    ltem render   [FILE]   HTML of all tables
+    ltem validate [FILE]   validation problems; exit status 1 if there are errors
+    ltem fmt      [FILE]   rewrite the input with every table in canonical form
+    ltem write    [FILE]   canonical LTEM of table models given as JSON
+    ltem from-html [FILE] [--header-rows N] [--plain]   convert HTML tables to LTEM
+
+Input is read from FILE or standard input. `parse`, `render`, `validate --json` and
+`write` follow the command-line protocol of the LTEM conformance tests.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from .fromhtml import from_html
+from .html import render_html
+from .model import Table
+from .parser import find_tables
+from .writer import write
+
+
+def _read(path: str | None) -> str:
+    """Read the input with every line ending turned into "\\n", for stdin as for files."""
+    if path in (None, "-"):
+        text = sys.stdin.read()
+    else:
+        with open(path, encoding="utf-8", newline="") as f:
+            text = f.read()
+    return text.replace("\r\n", "\n").replace("\r", "\n")
+
+
+def _fmt(text: str) -> str:
+    """Replace each table in the text by its canonical form, keeping everything else.
+
+    The table keeps the indentation of its first line, so a table in a list item stays there.
+    """
+    lines = text.split("\n")
+    for result in reversed(find_tables(text)):
+        assert result.table is not None
+        first = lines[result.start]
+        indent = first[: len(first) - len(first.lstrip(" \t"))]
+        lines[result.start:result.end] = [indent + line for line in write(result.table).split("\n")]
+    return "\n".join(lines)
+
+
+def _write(text: str) -> int:
+    """Print the canonical form of table models: one model, or {"tables": [...]}."""
+    try:
+        data = json.loads(text)
+        items = data["tables"] if isinstance(data, dict) and "tables" in data and "version" not in data else [data]
+        tables = [Table.from_dict(d) for d in items]
+    except (ValueError, KeyError, TypeError) as e:
+        print(f"ltem: error: not a table model: {e}", file=sys.stderr)
+        return 2
+    for n, table in enumerate(tables, 1):
+        problems = table.check()
+        if problems:
+            print(f"ltem: table {n}: invalid table model: {'; '.join(problems)}", file=sys.stderr)
+            return 1
+    print("\n\n".join(write(t) for t in tables))
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(prog="ltem", description="LTEM tables: parse, render, validate, format.")
+    sub = p.add_subparsers(dest="command", required=True)
+    for name in ("parse", "render", "validate", "fmt", "write", "from-html"):
+        sp = sub.add_parser(name)
+        sp.add_argument("file", nargs="?")
+        if name == "validate":
+            sp.add_argument("--json", action="store_true", help="print problems as JSON")
+        if name == "from-html":
+            sp.add_argument("--header-rows", type=int, default=None,
+                            help="number of header rows (default: <thead>, leading <th> rows, or 1; "
+                                 "0: no header)")
+            sp.add_argument("--plain", action="store_true", help="cell text only, without Markdown formatting")
+    args = p.parse_args(argv)
+    text = _read(args.file)
+    if args.command == "write":
+        return _write(text)
+    if args.command == "from-html":
+        converted = from_html(text, args.header_rows, "plain" if args.plain else "markdown")
+        for n, item in enumerate(converted, 1):
+            for w in item.warnings:
+                print(f"table {n}: {w}", file=sys.stderr)
+        print("\n\n".join(write(item.table) for item in converted))
+        return 0
+    results = find_tables(text)
+
+    if args.command == "parse":
+        json.dump({"tables": [r.table.to_dict() for r in results]}, sys.stdout, ensure_ascii=False, indent=2)
+        print()
+        return 0
+    if args.command == "render":
+        print("\n".join(render_html(r.table) for r in results))
+        return 0
+    if args.command == "validate":
+        problems = [pr for r in results for pr in r.problems]
+        if args.json:
+            json.dump({"errors": [{**pr.to_dict(), "severity": pr.severity} for pr in problems]}, sys.stdout,
+                      indent=2)
+            print()
+        else:
+            for n, r in enumerate(results, 1):
+                for pr in r.problems:
+                    print(f"table {n}: {pr}")
+        return 1 if any(pr.severity == "error" for pr in problems) else 0
+    sys.stdout.write(_fmt(text))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())