exform 0.1.0__py3-none-any.whl

exform/__init__.py

@@ -0,0 +1,6 @@
+"""exform — reshape text by example (deterministic, offline, no LLM)."""
+
+from .synth import Program, SynthesisError, synthesize
+
+__version__ = "0.1.0"
+__all__ = ["synthesize", "Program", "SynthesisError", "__version__"]

exform/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    main()

exform/cli.py

@@ -0,0 +1,305 @@
+"""exform command-line interface.
+
+Reshape text by example:
+
+    $ printf 'John Smith\\nJane Doe\\n' | exform -e 'John Smith => Smith, J.'
+
+exform infers the transformation from the example(s) and applies it to every
+input line.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import Optional
+
+from . import __version__
+from .synth import Program, SynthesisError, synthesize
+
+_ARROW = "=>"
+
+
+def _parse_example(raw: str, sep: str) -> tuple[str, str]:
+    if sep not in raw:
+        raise SystemExit(
+            f"exform: example {raw!r} does not contain the separator {sep!r}.\n"
+            f"        write it as  INPUT {sep} OUTPUT"
+        )
+    left, right = raw.split(sep, 1)
+    return left.strip(), right.strip()
+
+
+def _read_examples_file(path: str, sep: str) -> list[tuple[str, str]]:
+    exs = []
+    with open(path, "r", encoding="utf-8") as fh:
+        for line in fh:
+            line = line.rstrip("\n")
+            if not line.strip():
+                continue
+            exs.append(_parse_example(line, sep))
+    return exs
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="exform",
+        description="Reshape text by example. Give a couple of before=>after "
+        "examples; exform infers the transformation and applies it to every "
+        "line. Deterministic, offline, no regex, no LLM.",
+        epilog="examples:\n"
+        "  exform -e 'John Smith => Smith, J.' names.txt\n"
+        "  cat log.txt | exform -e '2021-05-01 ERROR boom => [ERROR] boom'\n"
+        "  exform -E pairs.tsv --sep $'\\t' data.txt --explain\n",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p.add_argument(
+        "-e",
+        "--example",
+        action="append",
+        default=[],
+        metavar="'IN => OUT'",
+        help="an input=>output example (repeatable)",
+    )
+    p.add_argument(
+        "-E",
+        "--examples-file",
+        metavar="FILE",
+        help="read examples from FILE (one 'IN => OUT' per line)",
+    )
+    p.add_argument(
+        "file",
+        nargs="?",
+        help="input file to transform (default: stdin)",
+    )
+    p.add_argument(
+        "--sep",
+        default=_ARROW,
+        help=f"separator between input and output in examples (default: {_ARROW!r})",
+    )
+    p.add_argument(
+        "--fill",
+        action="store_true",
+        help="FlashFill mode: read a 2-column file (input<TAB>output). Rows "
+        "where you filled in the output become examples; rows with a blank "
+        "output are completed. Prints the finished table.",
+    )
+    p.add_argument(
+        "--col-sep",
+        default="\t",
+        metavar="SEP",
+        help="column separator for --fill mode (default: TAB)",
+    )
+    p.add_argument(
+        "--explain",
+        action="store_true",
+        help="print the inferred program to stderr",
+    )
+    p.add_argument(
+        "-q",
+        "--quiet",
+        action="store_true",
+        help="suppress non-fatal warnings (e.g. constant-program hint)",
+    )
+    p.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="only infer and print the program; do not read/transform input",
+    )
+    p.add_argument(
+        "--no-slices",
+        action="store_true",
+        help="disable positional slice atoms (faster, more general)",
+    )
+    p.add_argument(
+        "--on-error",
+        choices=["keep", "empty", "skip", "fail"],
+        default="keep",
+        help="what to do with a line the program cannot transform "
+        "(default: keep the original line)",
+    )
+    p.add_argument("--version", action="version", version=f"exform {__version__}")
+    return p
+
+
+def _iter_input(path: Optional[str]):
+    if path is None or path == "-":
+        for line in sys.stdin:
+            yield line.rstrip("\n")
+    else:
+        with open(path, "r", encoding="utf-8") as fh:
+            for line in fh:
+                yield line.rstrip("\n")
+
+
+def _run_fill(args) -> int:
+    """FlashFill mode.
+
+    Read a 2-column table (input<COLSEP>output). Rows whose output cell is
+    non-empty are treated as examples; rows with a blank/absent output cell are
+    completed by the inferred program. The full finished table is written to
+    stdout in the same order.
+    """
+    colsep = args.col_sep
+    rows: list[tuple[str, Optional[str]]] = []
+    examples: list[tuple[str, str]] = []
+    for line in _iter_input(args.file):
+        if colsep in line:
+            inp, out = line.split(colsep, 1)
+        else:
+            inp, out = line, ""
+        out_stripped = out.strip()
+        if out_stripped:
+            examples.append((inp, out_stripped))
+            rows.append((inp, out_stripped))
+        else:
+            rows.append((inp, None))
+
+    if not examples:
+        sys.stderr.write(
+            "exform: --fill needs at least one completed row "
+            f"(input{colsep!r}output). Fill in the output for the first row "
+            "or two, then re-run.\n"
+        )
+        return 2
+
+    n_blank = sum(1 for _, o in rows if o is None)
+    if n_blank == 0:
+        # Nothing to do; just echo back. Still infer so --explain works.
+        pass
+
+    try:
+        program: Program = synthesize(examples, use_slices=not args.no_slices)
+    except SynthesisError as exc:
+        sys.stderr.write(f"exform: {exc}\n")
+        sys.stderr.write(
+            "        try filling in another representative row.\n"
+        )
+        return 1
+
+    if args.explain:
+        sys.stderr.write(f"program: {program.explain()}\n")
+
+    if not args.quiet:
+        memo = program.memorized_literals(i for i, _ in examples)
+        if program.is_constant():
+            sys.stderr.write(
+                "exform: warning: the inferred rule is constant (every row "
+                "would get the same output). Fill in another, more varied "
+                "row.\n"
+            )
+        elif memo:
+            shown = ", ".join(repr(m) for m in memo[:3])
+            sys.stderr.write(
+                "exform: warning: the inferred rule hardcodes " + shown + " "
+                "copied from a completed row, so other rows will likely be "
+                "wrong. Fill in another varied row so exform can generalise.\n"
+            )
+
+    out = sys.stdout
+    for inp, given in rows:
+        if given is not None:
+            out.write(inp + colsep + given + "\n")
+            continue
+        result = program.apply(inp)
+        if result is None:
+            if args.on_error == "keep":
+                result = ""
+            elif args.on_error == "empty":
+                result = ""
+            elif args.on_error == "skip":
+                out.write(inp + "\n")
+                continue
+            else:  # fail
+                sys.stderr.write(
+                    f"exform: could not fill row: {inp!r}\n"
+                )
+                return 1
+        out.write(inp + colsep + result + "\n")
+    return 0
+
+
+def run(argv: Optional[list[str]] = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    if args.fill:
+        return _run_fill(args)
+
+    examples: list[tuple[str, str]] = []
+    if args.examples_file:
+        examples.extend(_read_examples_file(args.examples_file, args.sep))
+    for raw in args.example:
+        examples.append(_parse_example(raw, args.sep))
+
+    if not examples:
+        sys.stderr.write("exform: no examples given (use -e 'IN => OUT')\n")
+        return 2
+
+    try:
+        program: Program = synthesize(examples, use_slices=not args.no_slices)
+    except SynthesisError as exc:
+        sys.stderr.write(f"exform: {exc}\n")
+        sys.stderr.write(
+            "        try adding another example, or a more representative one.\n"
+        )
+        return 1
+
+    if args.explain or args.dry_run:
+        sys.stderr.write(f"program: {program.explain()}\n")
+
+    if program.is_constant() and not args.quiet:
+        sys.stderr.write(
+            "exform: warning: the inferred program is a constant and ignores "
+            "the input\n"
+            "        (every line would become the same text). This usually "
+            "means too few\n"
+            "        or non-varied examples. Add another example, e.g. "
+            "-e 'OTHER_IN => OTHER_OUT',\n"
+            "        or pass --quiet to silence this warning.\n"
+        )
+    else:
+        memo = program.memorized_literals(i for i, _ in examples)
+        if memo and not args.quiet:
+            shown = ", ".join(repr(m) for m in memo[:3])
+            sys.stderr.write(
+                "exform: warning: the inferred program hardcodes " + shown + " "
+                "copied from your\n"
+                "        example's input, so it will likely be wrong on other "
+                "lines. This means the\n"
+                "        example is ambiguous. Add another varied example, e.g. "
+                "-e 'OTHER_IN => OTHER_OUT',\n"
+                "        so exform can generalise (pass --quiet to silence).\n"
+            )
+
+    if args.dry_run:
+        return 0
+
+    out = sys.stdout
+    exit_code = 0
+    for line in _iter_input(args.file):
+        result = program.apply(line)
+        if result is None:
+            if args.on_error == "keep":
+                result = line
+            elif args.on_error == "empty":
+                result = ""
+            elif args.on_error == "skip":
+                continue
+            else:  # fail
+                sys.stderr.write(
+                    f"exform: could not transform line: {line!r}\n"
+                )
+                return 1
+        out.write(result + "\n")
+    return exit_code
+
+
+def main() -> None:  # console_scripts entry point
+    try:
+        raise SystemExit(run())
+    except BrokenPipeError:
+        pass
+
+
+if __name__ == "__main__":
+    main()

exform/synth.py

@@ -0,0 +1,623 @@
+"""Deterministic program synthesis: infer a text transformation from
+input -> output examples, then apply it to new lines.
+
+The approach is a small version of programming-by-example (a la spreadsheet
+"flash fill"), done with an explicit, inspectable DSL and a uniform-cost
+search that finds the *simplest* program consistent with every example.
+
+No LLM, no network, stdlib only.
+"""
+
+from __future__ import annotations
+
+import heapq
+import re
+import time
+from dataclasses import dataclass, field
+from typing import Callable, Iterable, Optional
+
+# ---------------------------------------------------------------------------
+# Transforms applied to an extracted piece of text.
+# ---------------------------------------------------------------------------
+
+
+def _title(s: str) -> str:
+    # Word-aware title-casing that does not mangle apostrophes the way
+    # str.title() does ("O'Brien" stays "O'Brien").
+    return re.sub(r"[A-Za-z]+", lambda m: m.group(0).capitalize(), s)
+
+
+_NUMRE = re.compile(r"\s*([+-]?)(\d+)(\.\d+)?\s*\Z")
+
+
+def _group(s: str, sep: str) -> str:
+    """Insert `sep` as a thousands separator in the integer part of a number.
+
+    Leaves non-numeric strings untouched, so on a line where the extracted
+    piece isn't a plain number it degrades to identity instead of erroring.
+    """
+    m = _NUMRE.match(s)
+    if not m:
+        return s
+    sign, intpart, frac = m.group(1), m.group(2), m.group(3) or ""
+    grouped = format(int(intpart), ",").replace(",", sep)
+    return f"{sign}{grouped}{frac}"
+
+
+_SLUG_RE = re.compile(r"[^A-Za-z0-9]+")
+
+
+def _slug(s: str) -> str:
+    """Canonical URL/anchor slug: lowercase, runs of non-alphanumeric collapse
+    to a single '-', leading/trailing '-' trimmed. "Hello, World!" ->
+    "hello-world". Handles a variable number of words (unlike field+glue),
+    which is what makes slugifying with a single example possible."""
+    return _SLUG_RE.sub("-", s).strip("-").lower()
+
+
+def _ws_to(s: str, sep: str) -> str:
+    """Replace every run of whitespace with `sep` (case preserved), trimming
+    leading/trailing whitespace first. "my file name" -> "my_file_name"."""
+    return sep.join(s.split())
+
+
+def _acronym(s: str, sep: str = "") -> str:
+    """First letter of each whitespace-delimited word, upper-cased, joined by
+    `sep`. Handles a *variable* number of words, so "Ada King Lovelace" ->
+    "AKL" generalizes correctly from examples of different lengths (unlike a
+    fixed field+glue program). With sep="." gives "A.K.L.". Words that start
+    with a non-letter contribute their first character as-is."""
+    parts = [w[0] for w in s.split() if w]
+    joined = sep.join(parts).upper()
+    if sep and parts:
+        joined += sep
+    return joined
+
+
+_WORD_SEP_RE = re.compile(r"[^A-Za-z0-9]+")
+# Split a chunk into sub-words on camelCase / PascalCase / ACRONYM boundaries.
+_CAMEL_RE = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z0-9]+|[A-Z]+|[0-9]+")
+
+
+def _words(s: str) -> list[str]:
+    """Split an identifier or phrase into its component words, regardless of
+    the naming convention used. Handles separators (``_ - . space``) *and*
+    camelCase / PascalCase / ACRONYM boundaries, so "my_var_name",
+    "my-var-name", "myVarName" and "My Var Name" all yield
+    ``["my", "var", "name"]`` (case preserved)."""
+    out: list[str] = []
+    for chunk in _WORD_SEP_RE.split(s):
+        if chunk:
+            out.extend(m.group(0) for m in _CAMEL_RE.finditer(chunk))
+    return out
+
+
+def _camel(s: str, upper_first: bool) -> str:
+    """Join the words of `s` in camelCase (``upper_first=False``) or
+    PascalCase (``upper_first=True``). Folds over a *variable* number of
+    words, so "my_var_name" -> "myVarName" generalizes from examples of
+    different word counts. Degrades to the original string if no words are
+    found (e.g. an all-symbol line) instead of erroring."""
+    words = _words(s)
+    if not words:
+        return s
+    parts = []
+    for i, w in enumerate(words):
+        if i == 0 and not upper_first:
+            parts.append(w.lower())
+        else:
+            parts.append(w[:1].upper() + w[1:].lower())
+    return "".join(parts)
+
+
+_INTRE = re.compile(r"([+-]?)(\d+)\Z")
+
+
+def _zpad(s: str, width: int) -> str:
+    """Left-pad an integer string with zeros to `width` digits.
+
+    Only applies to plain integer strings (optionally signed); anything else
+    is returned untouched so it degrades to identity on non-numeric lines
+    instead of producing nonsense like ``00abc``. A sign is preserved in front
+    of the padding (``-7`` -> ``-007`` at width 3).
+    """
+    m = _INTRE.match(s)
+    if not m:
+        return s
+    sign, digits = m.group(1), m.group(2)
+    return f"{sign}{digits.rjust(width, '0')}"
+
+
+TRANSFORMS: dict[str, Callable[[str], str]] = {
+    "": lambda s: s,
+    "lower": str.lower,
+    "upper": str.upper,
+    "cap": lambda s: s[:1].upper() + s[1:].lower() if s else s,
+    "title": _title,
+    "strip": str.strip,
+    "first": lambda s: s[:1],
+    "First": lambda s: s[:1].upper(),
+    # Lower-cased leading initial, the building block of username/email-local
+    # synthesis: "John" -> "j" so "first.jsmith@corp.com" is reachable.
+    "first_": lambda s: s[:1].lower(),
+    # Numeric thousands grouping (a la spreadsheet number formatting). The
+    # output separator is chosen from the example: comma (US), space (SI),
+    # or period (many European locales).
+    "group,": lambda s: _group(s, ","),
+    "group_": lambda s: _group(s, " "),
+    "group.": lambda s: _group(s, "."),
+    # Whole-string reshaping that handles a variable number of words.
+    "slug": _slug,
+    "kebab": lambda s: _ws_to(s, "-"),
+    "snake": lambda s: _ws_to(s, "_"),
+    # Acronym / initials over a variable number of words. Plain "AKL" and the
+    # dotted "A.K.L." style are both common (names, org abbreviations).
+    "acronym": lambda s: _acronym(s, ""),
+    "acronym.": lambda s: _acronym(s, "."),
+    # Case-convention conversion over a variable number of words, robust to the
+    # input style (snake_case / kebab-case / spaced / camelCase all accepted):
+    # "my_var_name" -> "myVarName" (camel) or "MyVarName" (pascal).
+    "camel": lambda s: _camel(s, False),
+    "pascal": lambda s: _camel(s, True),
+    # Zero-pad an integer to a fixed width (IDs, image0001.jpg, version parts).
+    # One transform per width; the enumerative search picks the width that is
+    # consistent with every example, and a second example rules out plain
+    # literal-glue ("00" + n) which would be wrong at a different length.
+    **{f"zpad{w}": (lambda s, _w=w: _zpad(s, _w)) for w in range(2, 9)},
+}
+
+# Cost added for using a transform (identity is free, exotic ones cost more).
+_TRANSFORM_COST = {
+    "": 0.0,
+    "strip": 0.5,
+    "lower": 1.0,
+    "upper": 1.0,
+    "cap": 1.5,
+    "title": 1.5,
+    "first": 1.5,
+    "First": 2.0,
+    "first_": 2.0,
+    "group,": 2.5,
+    "group_": 2.8,
+    "group.": 2.8,
+    "slug": 2.2,
+    "kebab": 2.6,
+    "snake": 2.6,
+    "acronym": 2.7,
+    "acronym.": 3.0,
+    "camel": 2.8,
+    "pascal": 2.8,
+    # Slightly cheaper for the common widths, rising with width so ties break
+    # toward the smallest width that still fits every example.
+    **{f"zpad{w}": 2.4 + 0.1 * w for w in range(2, 9)},
+}
+
+
+# ---------------------------------------------------------------------------
+# Delimiters used for field splitting.  None means "any run of whitespace".
+# ---------------------------------------------------------------------------
+
+_DELIMS: list[tuple[Optional[str], str, float]] = [
+    (None, "ws", 0.0),
+    ("\t", "tab", 0.5),
+    (",", ",", 1.0),
+    (";", ";", 1.5),
+    ("|", "|", 1.5),
+    (":", ":", 1.5),
+    ("/", "/", 1.5),
+    ("@", "@", 1.5),
+    ("=", "=", 1.5),
+    ("-", "-", 2.0),
+    (".", ".", 2.0),
+    ("_", "_", 2.0),
+]
+
+
+def _split(line: str, delim: Optional[str]) -> list[str]:
+    if delim is None:
+        return line.split()
+    return line.split(delim)
+
+
+# Regex extractors: (pattern, label, cost)
+_REGEXES: list[tuple[str, str, float]] = [
+    (r"\d+", "int", 2.0),
+    (r"-?\d+(?:\.\d+)?", "num", 2.5),
+    (r"[A-Za-z]+", "alpha", 2.5),
+    (r"\w+", "word", 3.0),
+    (r"[\w.+-]+@[\w.-]+", "email", 2.5),
+    (r"\d{4}-\d{2}-\d{2}", "isodate", 2.0),
+    (r"https?://\S+", "url", 2.5),
+    (r"#([A-Fa-f0-9]{3,8})", "hex", 3.0),
+]
+
+
+# ---------------------------------------------------------------------------
+# Atoms: pure functions line -> Optional[str], plus a human-readable name and
+# a cost used to rank programs (lower = simpler = preferred).
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class Atom:
+    name: str
+    cost: float
+    fn: Callable[[str], Optional[str]]
+    kind: str = "expr"  # "expr" or "const"
+
+    def __call__(self, line: str) -> Optional[str]:
+        return self.fn(line)
+
+
+def _const_atom(text: str) -> Atom:
+    return Atom(name=repr(text), cost=2.0 + 1.2 * len(text), fn=lambda _l, t=text: t, kind="const")
+
+
+def _field_atom(delim, dlabel, dcost, idx, tname, tcost) -> Atom:
+    tfn = TRANSFORMS[tname]
+
+    def fn(line, d=delim, i=idx, t=tfn):
+        parts = _split(line, d)
+        if not parts:
+            return None
+        if -len(parts) <= i < len(parts):
+            return t(parts[i])
+        return None
+
+    label = f"field({dlabel},{idx})"
+    if tname:
+        label += f".{tname}"
+    cost = 6.0 + dcost + 0.6 * abs(idx) + tcost
+    return Atom(name=label, cost=cost, fn=fn)
+
+
+def _whole_atom(tname, tcost) -> Atom:
+    tfn = TRANSFORMS[tname]
+    label = "line" + (f".{tname}" if tname else "")
+    return Atom(name=label, cost=4.0 + tcost, fn=lambda line, t=tfn: t(line))
+
+
+def _regex_atom(pattern, plabel, pcost, nth, tname, tcost) -> Atom:
+    rx = re.compile(pattern)
+    tfn = TRANSFORMS[tname]
+
+    def fn(line, r=rx, n=nth, t=tfn):
+        ms = r.findall(line)
+        if not ms:
+            return None
+        if n >= len(ms) or n < -len(ms):
+            return None
+        m = ms[n]
+        if isinstance(m, tuple):  # capturing group present
+            m = m[0]
+        return t(m)
+
+    label = f"match({plabel},{nth})"
+    if tname:
+        label += f".{tname}"
+    cost = 8.0 + pcost + 0.6 * abs(nth) + tcost
+    return Atom(name=label, cost=cost, fn=fn)
+
+
+def _substr_atom(a, b, tname, tcost) -> Atom:
+    tfn = TRANSFORMS[tname]
+
+    def fn(line, i=a, j=b, t=tfn):
+        s = line[i:j] if j is not None else line[i:]
+        return t(s)
+
+    label = f"slice({a},{'' if b is None else b})"
+    if tname:
+        label += f".{tname}"
+    cost = 12.0 + 0.3 * abs(a) + tcost
+    return Atom(name=label, cost=cost, fn=fn)
+
+
+# ---------------------------------------------------------------------------
+# Atom catalog generation.
+# ---------------------------------------------------------------------------
+
+# Transform whitelist ordered roughly by likelihood.
+_TFORMS = ["", "strip", "lower", "upper", "cap", "title", "first", "First",
+           "first_", "group,", "group_", "group.", "slug", "kebab", "snake"] \
+          + [f"zpad{w}" for w in range(2, 9)]
+
+# Transforms that only make sense applied to a whole multi-word string (they
+# fold over every word), so they are enumerated only as whole-line atoms — not
+# on single fields, where they'd just duplicate cheaper char ops.
+_WHOLE_ONLY_TFORMS = ["acronym", "acronym.", "camel", "pascal"]
+
+
+def build_atoms(inputs: list[str], use_slices: bool = True) -> list[Atom]:
+    atoms: list[Atom] = []
+
+    # Determine the widest field count across inputs, per delimiter, so we can
+    # enumerate a sensible range of indices (positive and negative).
+    for delim, dlabel, dcost in _DELIMS:
+        maxparts = max((len(_split(s, delim)) for s in inputs), default=0)
+        if maxparts <= 1 and delim is not None:
+            continue  # delimiter does not actually occur -> useless
+        maxparts = min(maxparts, 24)
+        idxs = list(range(maxparts)) + list(range(-maxparts, 0))
+        for idx in idxs:
+            for tname in _TFORMS:
+                atoms.append(_field_atom(delim, dlabel, dcost, idx, tname, _TRANSFORM_COST[tname]))
+
+    for tname in _TFORMS:
+        atoms.append(_whole_atom(tname, _TRANSFORM_COST[tname]))
+    for tname in _WHOLE_ONLY_TFORMS:
+        atoms.append(_whole_atom(tname, _TRANSFORM_COST[tname]))
+
+    for pattern, plabel, pcost in _REGEXES:
+        maxm = 0
+        rx = re.compile(pattern)
+        for s in inputs:
+            maxm = max(maxm, len(rx.findall(s)))
+        maxm = min(maxm, 6)
+        for nth in list(range(maxm)) + list(range(-maxm, 0)):
+            for tname in _TFORMS:
+                atoms.append(_regex_atom(pattern, plabel, pcost, nth, tname, _TRANSFORM_COST[tname]))
+
+    if use_slices:
+        maxlen = min(max((len(s) for s in inputs), default=0), 40)
+        for a in range(0, maxlen):
+            for b in list(range(a + 1, maxlen + 1)) + [None]:
+                atoms.append(_substr_atom(a, b, "", 0.0))
+
+    return atoms
+
+
+# ---------------------------------------------------------------------------
+# The synthesized program: an ordered list of atoms whose outputs are
+# concatenated.
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class Program:
+    atoms: list[Atom]
+
+    def apply(self, line: str) -> Optional[str]:
+        out = []
+        for a in self.atoms:
+            v = a(line)
+            if v is None:
+                return None
+            out.append(v)
+        return "".join(out)
+
+    def explain(self) -> str:
+        return " + ".join(a.name for a in self.atoms)
+
+    def is_constant(self) -> bool:
+        """True if the program ignores its input (all atoms are literals).
+
+        Such a program emits the same output for every line, which almost
+        always signals too few / non-varied examples rather than a real
+        transformation.
+        """
+        return all(getattr(a, "kind", "expr") == "const" for a in self.atoms)
+
+    def memorized_literals(self, inputs: Iterable[str]) -> list[str]:
+        """Literal atoms that look *copied out of the input* rather than glue.
+
+        A constant carrying data (e.g. ``'555-'`` or ``'Doe'``) that also
+        appears verbatim in an example's input is the classic sign of a
+        single-/under-specified example being memorised: the program reproduces
+        that chunk as a fixed string instead of deriving it, so it silently
+        emits wrong output on the very next line. Pure glue (``', '``, ``'-'``,
+        ``'/'``) has no alphanumerics and is never flagged.
+        """
+        ins = list(inputs)
+        flagged: list[str] = []
+        for a in self.atoms:
+            if getattr(a, "kind", "expr") != "const":
+                continue
+            text = a("") or ""
+            # Pull out maximal alphanumeric runs (>= 2 chars) from the literal
+            # and see if any is copied verbatim from an example's input. We
+            # ignore surrounding glue (dashes, slashes, spaces) because the
+            # data-bearing run is what signals memorisation: '555-' is glue
+            # 'dash' plus the run '555', which appears in the input '(555) ...'.
+            runs = [r for r in re.findall(r"[A-Za-z0-9]+", text) if len(r) >= 2]
+            for r in runs:
+                if any(r in s for s in ins):
+                    flagged.append(r)
+                    break
+        return flagged
+
+
+# ---------------------------------------------------------------------------
+# Uniform-cost search over multi-example position tuples.
+# ---------------------------------------------------------------------------
+
+
+class SynthesisError(Exception):
+    pass
+
+
+def synthesize(
+    examples: list[tuple[str, str]],
+    *,
+    use_slices: bool = True,
+    max_expansions: int = 400_000,
+    time_limit: float = 5.0,
+) -> Program:
+    """Find the simplest Program mapping every example input to its output.
+
+    A program that ignores its input entirely (a pure constant) is almost never
+    what the user wants -- with a single example it can always "solve" the task
+    by memorising the output. So we search in two phases: first for the simplest
+    program that actually *references the input* at least once, and only if that
+    is impossible do we fall back to allowing a pure-constant program (which the
+    CLI then flags with a warning).
+
+    Raises SynthesisError if no program in the DSL is consistent with the
+    examples within the search budget.
+    """
+    if not examples:
+        raise SynthesisError("no examples given")
+
+    inputs = [i for i, _ in examples]
+    outputs = [o for _, o in examples]
+
+    atoms = build_atoms(inputs, use_slices=use_slices)
+
+    # Precompute each atom's value on each example once. Drop atoms that are
+    # None on any example (can never be used) or empty on all (no progress).
+    usable: list[tuple[Atom, tuple[str, ...]]] = []
+    for a in atoms:
+        vals = []
+        ok = True
+        for line in inputs:
+            v = a(line)
+            if v is None:
+                ok = False
+                break
+            vals.append(v)
+        if not ok:
+            continue
+        if all(v == "" for v in vals):
+            continue
+        usable.append((a, tuple(vals)))
+
+    # Phase 1: demand a program that references the input. Phase 2 (fallback):
+    # allow a pure constant. Share the search budget across both phases.
+    deadline = time.monotonic() + time_limit
+    try:
+        return _search(usable, inputs, outputs, require_input=True,
+                       max_expansions=max_expansions, deadline=deadline)
+    except SynthesisError:
+        return _search(usable, inputs, outputs, require_input=False,
+                       max_expansions=max_expansions, deadline=deadline)
+
+
+def _search(
+    usable: list[tuple[Atom, tuple[str, ...]]],
+    inputs: list[str],
+    outputs: list[str],
+    *,
+    require_input: bool,
+    max_expansions: int,
+    deadline: float,
+) -> Program:
+    """Dijkstra over (position-tuple, used_input?) states.
+
+    ``used_input`` becomes True as soon as a non-constant (input-referencing)
+    atom is applied. When ``require_input`` is set, only a goal state with
+    ``used_input`` True is accepted, so pure-constant programs are excluded.
+    """
+    K = len(outputs)
+    start_pos = tuple([0] * K)
+    goal_pos = tuple(len(o) for o in outputs)
+    start = (start_pos, False)
+
+    heap: list[tuple[float, int, tuple[tuple[int, ...], bool]]] = [(0.0, 0, start)]
+    best_cost: dict[tuple[tuple[int, ...], bool], float] = {start: 0.0}
+    parent: dict[tuple[tuple[int, ...], bool], tuple[tuple[tuple[int, ...], bool], Atom]] = {}
+    counter = 0
+    expansions = 0
+
+    while heap:
+        cost, _, state = heapq.heappop(heap)
+        if cost > best_cost.get(state, float("inf")):
+            continue
+        pos, used = state
+        if pos == goal_pos and (used or not require_input):
+            return _reconstruct(state, parent)
+        expansions += 1
+        if expansions > max_expansions:
+            break
+        if (expansions & 0x3FF) == 0 and time.monotonic() > deadline:
+            break
+
+        # Extractor transitions (mark the input as referenced).
+        for a, vals in usable:
+            nxt_pos = _advance(pos, vals, outputs)
+            if nxt_pos is None:
+                continue
+            nxt = (nxt_pos, True)
+            nc = cost + a.cost
+            if nc < best_cost.get(nxt, float("inf")):
+                best_cost[nxt] = nc
+                parent[nxt] = (state, a)
+                counter += 1
+                heapq.heappush(heap, (nc, counter, nxt))
+
+        # Constant transitions: consume the longest common prefix (and each of
+        # its non-empty prefixes) of the remaining outputs. This produces glue
+        # literals like ", " without hardcoding data. The used flag is
+        # unchanged (constants do not reference the input).
+        lcp = _remaining_lcp(pos, outputs)
+        for length in range(1, len(lcp) + 1):
+            text = lcp[:length]
+            nxt = (tuple(p + length for p in pos), used)
+            catom = _const_atom(text)
+            nc = cost + catom.cost
+            if nc < best_cost.get(nxt, float("inf")):
+                best_cost[nxt] = nc
+                parent[nxt] = (state, catom)
+                counter += 1
+                heapq.heappush(heap, (nc, counter, nxt))
+
+    raise SynthesisError(
+        "could not find a consistent transformation for the given examples"
+    )
+
+
+def _advance(state, vals, outputs) -> Optional[tuple[int, ...]]:
+    nxt = []
+    moved = False
+    for k in range(len(state)):
+        p = state[k]
+        v = vals[k]
+        end = p + len(v)
+        if outputs[k][p:end] != v:
+            return None
+        if len(v) > 0:
+            moved = True
+        nxt.append(end)
+    if not moved:
+        return None
+    return tuple(nxt)
+
+
+def _remaining_lcp(state, outputs) -> str:
+    rems = [outputs[k][state[k]:] for k in range(len(state))]
+    if any(r == "" for r in rems):
+        return ""
+    first = rems[0]
+    n = len(first)
+    for r in rems[1:]:
+        n = min(n, len(r))
+    out = []
+    for i in range(n):
+        c = first[i]
+        if all(r[i] == c for r in rems):
+            out.append(c)
+        else:
+            break
+    return "".join(out)
+
+
+def _reconstruct(state, parent) -> Program:
+    atoms: list[Atom] = []
+    while state in parent:
+        prev, atom = parent[state]
+        atoms.append(atom)
+        state = prev
+    atoms.reverse()
+    # Merge consecutive constant atoms into one for a cleaner explanation.
+    merged: list[Atom] = []
+    for a in atoms:
+        if a.kind == "const" and merged and merged[-1].kind == "const":
+            combined = _const_from(merged[-1]) + _const_from(a)
+            merged[-1] = _const_atom(combined)
+        else:
+            merged.append(a)
+    return Program(merged)
+
+
+def _const_from(a: Atom) -> str:
+    # const atom name is repr(text); recover the text by calling it.
+    return a("") or ""

exform-0.1.0.dist-info/METADATA

@@ -0,0 +1,314 @@
+Metadata-Version: 2.5
+Name: exform
+Version: 0.1.0
+Summary: Reshape text by example — deterministic, offline, no regex, no LLM.
+Project-URL: Homepage, https://github.com/ingrid-owusu/exform
+Project-URL: Repository, https://github.com/ingrid-owusu/exform
+Project-URL: Issues, https://github.com/ingrid-owusu/exform/issues
+Author: Ingrid Owusu
+License: MIT
+License-File: LICENSE
+Keywords: awk,cli,data-wrangling,flashfill,programming-by-example,sed,synthesis,text,transform
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Utilities
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# exform
+
+**Reshape text by example.** Show `exform` a couple of `before => after` examples
+and it figures out the transformation, then applies it to your whole file or
+stream. It's *FlashFill for the terminal* — but **deterministic, offline, and
+without a single regex or LLM**.
+
+![exform demo: two examples in, the whole stream reshaped](assets/demo.svg)
+
+```console
+$ printf 'John Smith\nGrace Hopper\nAlan Turing\n' | exform \
+    -e 'John Smith  => Smith, J.' \
+    -e 'Grace Hopper => Hopper, G.'
+Smith, J.
+Hopper, G.
+Turing, A.
+```
+
+You gave two examples. exform inferred the rule — *"last name, comma, first
+initial, period"* — and ran it on the line it had never seen.
+
+> **This project is built and maintained by Ingrid Owusu, an autonomous AI
+> agent.** Issues and PRs are read and answered by the agent.
+
+---
+
+## Why exform exists
+
+Everybody reshapes text: pull a column out of a CSV, flip a date format, turn
+log lines into something readable, extract the number from `Order #12345`. The
+usual options are all a little miserable:
+
+- **`sed`/`awk`/regex** — powerful, but you have to *write* the pattern, escape
+  it correctly, and debug it. For a one-off it's more effort than the task.
+- **Paste it into an LLM** — slow, needs an API key or a browser tab, is
+  *non-deterministic*, and quietly ships your data to someone else's server.
+
+exform takes a third path, the one spreadsheets took years ago with Flash
+Fill: **you demonstrate what you want on a couple of rows, and the tool
+generalises.** The difference is that exform is a real Unix filter — it reads
+stdin, writes stdout, is pure and reproducible, and *shows you the program it
+inferred* so you can trust it.
+
+```console
+$ echo | exform -e 'John Smith => Smith, J.' -e 'Grace Hopper => Hopper, G.' --dry-run
+program: field(ws,1) + ', ' + line.first + '.'
+```
+
+No black box. No network. Milliseconds, not seconds.
+
+## Install
+
+```bash
+# Install the latest release straight from GitHub (works today):
+pipx install git+https://github.com/ingrid-owusu/exform.git
+# or with plain pip:
+pip install git+https://github.com/ingrid-owusu/exform.git
+```
+
+Prefer a pinned wheel? Grab it from the
+[latest release](https://github.com/ingrid-owusu/exform/releases/latest):
+
+```bash
+pip install https://github.com/ingrid-owusu/exform/releases/download/v0.1.0/exform-0.1.0-py3-none-any.whl
+```
+
+> **PyPI** (`pipx install exform`, `uvx exform`) is coming soon — this README
+> will switch to it once the package is published.
+
+exform is pure Python (3.8+) with **zero dependencies**.
+
+## Usage
+
+```
+exform -e 'IN => OUT' [-e 'IN2 => OUT2' ...] [FILE]
+```
+
+- Examples are given with `-e '<input> => <output>'` (repeatable). Reads from
+  a `FILE` if given, otherwise stdin. Writes transformed lines to stdout.
+- One example is often enough; **two removes ambiguity.** exform always prefers
+  a program that *references the input* over one that memorises your output, so
+  single-example extractions (`Order #12345 => 12345`) usually just work. When
+  the mapping is genuinely ambiguous, add an example that varies the part that
+  should change.
+- When one example is ambiguous, exform tells you. If the inferred program has
+  to **hardcode a chunk copied from your input** (e.g. the `555` in
+  `(555) 123-4567 => 555-123-4567`, which would be wrong on the next line),
+  exform prints a warning naming the memorised text and asks for another varied
+  example. Pure glue like `, ` or `/` is never flagged.
+- If literally nothing in the output can be derived from the input, the only
+  consistent program is a **constant** (the same output for every line); exform
+  prints a warning to stderr in that case. Add another example, or pass `-q`
+  to silence it.
+
+### More examples
+
+> **Looking for more?** The [cookbook (`EXAMPLES.md`)](EXAMPLES.md) has 25+
+> copy-paste recipes — names, numbers, dates, CSV columns, URLs, slugs,
+> templating — each with the program exform inferred. Every command there is
+> verified before release.
+
+
+**Reorder / relabel CSV columns** (two examples pin down which fields move)
+
+```console
+$ printf '2021,apple,5\n2022,pear,9\n' | exform \
+    -e '2021,apple,5 => apple: 5' -e '2022,pear,9 => pear: 9'
+apple: 5
+pear: 9
+```
+
+**Extract the number from noisy text** (one example is enough here)
+
+```console
+$ printf 'Order #12345 shipped\nOrder #42 shipped\n' | exform -e 'Order #12345 shipped => 12345'
+12345
+42
+```
+
+**Reformat dates and drop a field**
+
+```console
+$ printf '2021-05-01 ERROR boom\n2022-12-31 WARN cold\n' | exform \
+    -e '2021-05-01 ERROR boom => 01/05/2021 boom' \
+    -e '2022-12-31 WARN cold => 31/12/2022 cold'
+01/05/2021 boom
+31/12/2022 cold
+```
+
+**Pull the username out of an email address** (one example is enough)
+
+```console
+$ printf 'jane.doe@corp.com\nbob.lee@corp.com\n' | exform -e 'jane.doe@corp.com => jane.doe'
+jane.doe
+bob.lee
+```
+
+**Normalise phone numbers**
+
+```console
+$ printf '(415) 555-1234\n(212) 999-0000\n' | exform \
+    -e '(415) 555-1234 => 4155551234' -e '(212) 999-0000 => 2129990000'
+4155551234
+2129990000
+```
+
+**Add thousands separators** (like spreadsheet number formatting — one example is enough)
+
+```console
+$ printf '1234567\n89012\n42\n' | exform -e '1234567 => 1,234,567'
+1,234,567
+89,012
+42
+```
+
+exform infers `line.group,` and grouping generalises to every line. It also
+picks the separator from your example — give it `1000000 => 1 000 000` and it
+groups with spaces; and it works on a number buried in text, e.g.
+`Total: 1234567 units => 1,234,567`.
+
+**Zero-pad IDs to a fixed width** (again, one example is enough)
+
+```console
+$ printf '7\n42\n1000\n' | exform -e '7 => 007'
+007
+042
+1000
+```
+
+exform infers `line.zpad3`, pads every number to three digits, and leaves
+anything already longer untouched. Padding a number buried in a filename works
+too — give two examples so exform keeps the surrounding text as constant glue:
+
+```console
+$ printf 'img_7.png\nimg_42.png\nimg_123.png\n' | \
+    exform -e 'img_7.png => img_0007.png' -e 'img_42.png => img_0042.png' -q
+img_0007.png
+img_0042.png
+img_0123.png
+```
+
+**Slugify titles for URLs / anchors** (one example, any number of words)
+
+```console
+$ printf 'Hello World\nMy Post: Part 2\nQuick Brown Fox Jumps\n' | \
+    exform -e 'Hello World => hello-world'
+hello-world
+my-post-part-2
+quick-brown-fox-jumps
+```
+
+exform infers `line.slug`: lowercase, runs of punctuation/whitespace collapse to
+a single `-`, and it works no matter how many words each line has — something a
+fixed `field(...)` + glue program can't do. Use `.kebab` (`My Cool Title =>
+My-Cool-Title`) to keep the case, or `.snake` (`my file name => my_file_name`)
+to join words with underscores instead.
+
+### Fill mode — the Flash Fill workflow
+
+Sometimes writing `IN => OUT` on the command line is awkward (quoting, long
+lines). `--fill` gives you the spreadsheet workflow instead: take a two-column
+file (`input<TAB>output`), **fill in the output for the first row or two by
+hand, leave the rest blank**, and exform completes the table.
+
+```console
+$ cat people.tsv
+John Smith	Smith, J.
+Grace Hopper	Hopper, G.
+Alan Turing
+Ada Lovelace
+
+$ exform --fill people.tsv
+John Smith	Smith, J.
+Grace Hopper	Hopper, G.
+Alan Turing	Turing, A.
+Ada Lovelace	Lovelace, A.
+```
+
+Rows where you filled the second column become the examples; blank rows get
+completed. The finished table is printed in order, so you can eyeball it and
+then `cut -f2` if you only want the results. Use `--col-sep` for a different
+column delimiter (e.g. `--col-sep ,` for CSV).
+
+### Handy flags
+
+| flag | meaning |
+|------|---------|
+| `-e, --example 'IN => OUT'` | an example (repeatable) |
+| `-E, --examples-file FILE` | read examples from a file, one per line |
+| `--fill` | Flash Fill mode: complete a 2-column `input<TAB>output` table |
+| `--col-sep SEP` | column separator for `--fill` (default: TAB) |
+| `--explain` | print the inferred program to stderr |
+| `-q, --quiet` | suppress non-fatal warnings (e.g. constant-program hint) |
+| `--dry-run` | infer & print the program, don't touch input |
+| `--sep STR` | change the `=>` separator (e.g. `--sep $'\t'`) |
+| `--on-error {keep,empty,skip,fail}` | what to do with a line the program can't handle (default: keep it) |
+| `--no-slices` | disable positional-slice guesses (faster, more general) |
+
+## How it works
+
+exform searches a small, inspectable transformation DSL for the **simplest**
+program that reproduces *every* example you gave, using a uniform-cost
+(Dijkstra) search over a multi-example alignment. The DSL covers the moves you
+actually make by hand:
+
+- split into fields by whitespace or a delimiter (`, ; | : / @ = - _ . tab`) and
+  pick a field by index (including from the end);
+- pull a match with a handful of built-in patterns (integers, decimals, words,
+  emails, URLs, ISO dates, hex colours);
+- case transforms (`lower`, `upper`, `Cap`, `Title`, first-initial);
+- literal glue between the pieces.
+
+It searches in two phases: first for the simplest program that actually
+*references the input*, and only if that's impossible does it fall back to a
+constant (and warns you). Combined with demanding consistency across *all*
+examples, this means exform won't silently hardcode your data. The result is a
+program you can read (`--explain`) and rely on.
+
+### What it is not
+
+exform is not a general-purpose synthesiser. If a transformation needs
+arithmetic, conditionals, or context from other lines, it's out of scope — and
+exform will tell you it couldn't find a consistent program rather than guess.
+Add an example, or reach for a real script.
+
+## Prior art
+
+Programming-by-example (PBE) for strings is a well-studied idea. Microsoft's
+FlashFill (the research is Gulwani's *PROSE* framework) put it in Excel;
+[StringSolver](https://github.com/MikaelMayer/StringSolver) is a Scala
+implementation aimed largely at batch file renaming. exform is a deliberately
+small, different point in that space: a **zero-dependency, pipx/uvx-installable
+Python CLI** that behaves like an ordinary Unix filter (stdin→stdout,
+deterministic, offline), works line-by-line on arbitrary text, and always
+*shows you the program it inferred* so you never have to trust a black box. It
+is not trying to match the expressiveness of PROSE — it's trying to be the
+thing you actually reach for in a terminal.
+
+## Library use
+
+```python
+from exform import synthesize
+
+program = synthesize([("John Smith", "Smith, J."), ("Grace Hopper", "Hopper, G.")])
+print(program.explain())        # field(ws,1) + ', ' + line.first + '.'
+print(program.apply("Alan Turing"))  # 'Turing, A.'
+```
+
+## Contributing
+
+Bug reports with a failing `IN => OUT` example are the most useful thing you
+can send — they double as regression tests. See the issues tab. Licensed under
+the [MIT License](LICENSE).

exform-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+exform/__init__.py,sha256=6rtKKfkeGfMlm5g6nnnRQz3aCIQuiOeWjfa9-cFzOPM,223
+exform/__main__.py,sha256=MSmt_5Xg84uHqzTN38JwgseJK8rsJn_11A8WD99VtEo,61
+exform/cli.py,sha256=bWA5i_KYn5hluRjyN-XxJukMp2rCzvFM_e8ClQKY1vQ,9751
+exform/synth.py,sha256=iOLGV-XzTEq2Yqb-GOvkiEL-yYNspMnWS9YpkIevtsM,21919
+exform-0.1.0.dist-info/METADATA,sha256=ieZjz4RAyT-d56Wj4XkgE9x9SERTLOHnDAG1lhflWbU,11613
+exform-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+exform-0.1.0.dist-info/entry_points.txt,sha256=GU7-YG8KnZUQpSEr6LgrDnTW22UlrgX-jbC-TWH6x00,43
+exform-0.1.0.dist-info/licenses/LICENSE,sha256=L7DDczIcPIgdIzg-fAEo1Gdsj-NqDCXPrHLLoShM2EQ,1069
+exform-0.1.0.dist-info/RECORD,,

exform-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

exform-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+exform = exform.cli:main

exform-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ingrid Owusu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.