meltify 0.1.0__py3-none-any.whl

meltify/__init__.py

@@ -0,0 +1,6 @@
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("meltify")
+except PackageNotFoundError:
+    __version__ = "0.0.0"

meltify/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from meltify.cli import main
+
+sys.exit(main())

meltify/cli.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import argparse
+import importlib
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+from types import ModuleType
+
+from meltify import __version__, config
+from meltify.evidence import MISSING, USAGE, Envelope
+from meltify.output import emit
+from meltify.safe import MissingTool
+
+# Each module exposes NAME, HELP, COLUMNS, add_arguments and run.
+# Heavy imports live inside run, so --help and --version start fast
+COMMANDS: list[str] = ["read", "ocr", "hidden", "media", "submit", "check", "brief", "doctor"]
+
+USAGE_ERROR = 2
+
+EXIT_CODES = """exit codes:
+  0  success
+  1  violations, failed checks or other errors
+  2  usage error or bad input
+  3  a needed engine, key, binary or base module is missing"""
+
+
+def _modules() -> list[ModuleType]:
+    return [importlib.import_module(f"meltify.commands.{name}") for name in COMMANDS]
+
+
+def _limit(raw: str) -> int:
+    n = int(raw)
+    if n < 0:
+        raise argparse.ArgumentTypeError(f"must be 0 or more, got {n}")
+    return n
+
+
+def build_parser(modules: Sequence[ModuleType]) -> argparse.ArgumentParser:
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("--json", action="store_true", help="print the full result envelope")
+    common.add_argument("--out", type=Path, help="output directory for artifacts")
+    common.add_argument("--config", type=Path, help="project config file instead of meltify.toml")
+    common.add_argument(
+        "--limit", type=_limit, default=50, help="rows shown in the table, 0 for all"
+    )
+
+    parser = argparse.ArgumentParser(
+        prog="meltify",
+        description="Melt files of any format into prompt-ready text, with a citation per item",
+        epilog=EXIT_CODES,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--version", action="version", version=f"meltify {__version__}")
+    sub = parser.add_subparsers(dest="command", metavar="COMMAND")
+    for m in modules:
+        p = sub.add_parser(
+            m.NAME,
+            help=m.HELP,
+            description=m.HELP,
+            epilog=EXIT_CODES,
+            formatter_class=argparse.RawDescriptionHelpFormatter,
+            parents=[common],
+        )
+        m.add_arguments(p)
+        p.set_defaults(_module=m)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    modules = _modules()
+    parser = build_parser(modules)
+    args = parser.parse_args(argv)
+    if not getattr(args, "_module", None):
+        parser.print_help(sys.stderr)
+        return USAGE_ERROR
+    module = args._module
+
+    overrides = {"out_dir": str(args.out)} if args.out is not None else None
+    try:
+        settings = config.load(overrides, project=args.config)
+    except config.ConfigError as e:
+        # Report it in the envelope, so --json callers still get valid JSON
+        env = Envelope(command=module.NAME, version=__version__)
+        env.error(USAGE, f"config: {e}")
+        emit(env, as_json=args.json, columns=module.COLUMNS, limit=args.limit)
+        return env.exit_code
+
+    try:
+        env = module.run(args, settings)
+    except MissingTool as e:
+        env = Envelope(command=module.NAME, version=__version__)
+        env.error(MISSING, str(e), e.hint)
+    except (FileNotFoundError, IsADirectoryError, ValueError) as e:
+        # Report bad input in the envelope, so --json callers still get valid JSON
+        env = Envelope(command=module.NAME, version=__version__)
+        env.error(USAGE, f"{type(e).__name__}: {e}")
+    emit(env, as_json=args.json, columns=env.columns or module.COLUMNS, limit=args.limit)
+    return env.exit_code

meltify/commands/brief.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import time
+from importlib.resources import files
+from pathlib import Path
+from typing import Any
+
+from meltify import __version__
+from meltify.evidence import USAGE, Envelope, Src, finding
+
+NAME = "brief"
+HELP = "quote a problem statement's conditions with line numbers, and track several problems"
+COLUMNS = ["kinds", "quote", "cite"]
+BOARD_COLUMNS = ["problem", "state", "points", "checks", "idle_min", "answer"]
+
+# Words that mark a sentence a solver shouldn't paraphrase, in English and Korean
+KINDS: dict[str, re.Pattern[str]] = {
+    "count": re.compile(
+        r"\d+\s*(연승|연속|회|번|times?|in a row|consecutive|wins?|attempts?|items?|개|명|건)"
+        r"|at least|at most|exactly|최소|최대|이상|이하|미만|초과",
+        re.I,
+    ),
+    "date": re.compile(r"\d{4}[-./]\d{1,2}[-./]\d{1,2}|기준일|as of|deadline|마감|기한|due", re.I),
+    "timezone": re.compile(r"\b(KST|UTC|GMT|PST|EST)\b|[+-]\d{2}:\d{2}|시간대|time ?zone", re.I),
+    "format": re.compile(
+        r"JSON|CSV|대문자|소문자|upper ?case|lower ?case|숫자만|digits? only|단어|words?\b"
+        r"|형식|format|쉼표|comma|소수점|decimal|템플릿|template|띄어쓰기|spaces?\b",
+        re.I,
+    ),
+    "priority": re.compile(
+        r"우선|priority|lowest|highest|가장 낮은|가장 높은|먼저|first"
+        r"|여러 개|multiple|if more than",
+        re.I,
+    ),
+    "limit": re.compile(
+        r"per (minute|second|hour)|분당|초당|\d+\s*분에|rate limit|제한"
+        r"|한 번만|only once|duplicate|중복",
+        re.I,
+    ),
+    "prohibition": re.compile(
+        r"금지|must not|do not|don't|never|않|없이|추측|guess|assume|불가", re.I
+    ),
+}
+
+
+def add_arguments(p: argparse.ArgumentParser) -> None:
+    p.add_argument("action", choices=["extract", "new", "board", "set"])
+    p.add_argument("target", nargs="?", help="statement file for extract, number for new and set")
+    p.add_argument("title", nargs="?", help="title for new")
+    p.add_argument("--from", dest="source", type=Path, help="statement file for new")
+    p.add_argument("--state", help="state for set, like todo, doing, done or skipped")
+    p.add_argument("--points", type=float, help="points for set")
+    p.add_argument("--answer", help="answer note for set")
+    p.add_argument("--force", action="store_true", help="rewrite an existing problem's NOTES.md")
+
+
+def extract(text: str, origin: str) -> list[dict[str, Any]]:
+    rows = []
+    for n, line in enumerate(text.splitlines(), start=1):
+        quote = line.strip()
+        if not quote:
+            continue
+        kinds = [k for k, rx in KINDS.items() if rx.search(quote)]
+        if kinds:
+            rows.append(finding(Src(origin, line=n), kinds=kinds, quote=quote))
+    return rows
+
+
+def _slug(title: str) -> str:
+    return re.sub(r"[^\w]+", "-", title).strip("-")[:40] or "problem"
+
+
+def _problem_dirs(root: Path, number: str) -> list[Path]:
+    prefix = f"{int(number):02d}-"
+    return [d for d in sorted(root.glob(f"{prefix}*")) if d.is_dir()]
+
+
+def _new(d: Path, number: str, title: str, source: Path | None, body: str) -> None:
+    statement = source.read_text("utf-8") if source else ""
+    conditions = extract(statement, str(source)) if source else []
+    listed = "\n".join(f"- [ ] {r['quote']}  `{r['cite']}`" for r in conditions)
+    notes = body.format(
+        number=number, title=title, statement=statement.strip(), conditions=listed or "-"
+    )
+    (d / "data").mkdir(parents=True, exist_ok=True)
+    (d / "NOTES.md").write_text(notes, encoding="utf-8")
+    status = d / "status.json"
+    if not status.exists():
+        status.write_text(json.dumps({"state": "todo", "points": 0, "answer": None}), "utf-8")
+
+
+def _board(dirs: list[Path], now: float) -> list[dict[str, Any]]:
+    rows = []
+    for d in dirs:
+        status = json.loads((d / "status.json").read_text("utf-8"))
+        notes = (d / "NOTES.md").read_text("utf-8")
+        done, todo = notes.count("- [x]"), notes.count("- [ ]")
+        newest = max(f.stat().st_mtime for f in d.rglob("*") if f.is_file())
+        rows.append(
+            finding(
+                Src(str(d / "NOTES.md")),
+                problem=d.name,
+                state=status.get("state"),
+                points=status.get("points"),
+                checks=f"{done}/{done + todo}",
+                idle_min=int((now - newest) / 60),
+                answer=status.get("answer"),
+            )
+        )
+    return rows
+
+
+def run(args: argparse.Namespace, settings: dict[str, Any]) -> Envelope:
+    env = Envelope(command=NAME, version=__version__)
+    conf = settings.get("brief", {})
+    root = Path(conf.get("root", "problems"))
+
+    if args.action == "extract":
+        if not args.target:
+            env.error(USAGE, "extract needs a statement file")
+            return env
+        path = Path(args.target)
+        env.inputs.append({"path": str(path)})
+        env.results = extract(path.read_text("utf-8"), str(path))
+        env.summary = (
+            f"{len(env.results)} condition lines. Quote them as written and check each one"
+        )
+        return env
+
+    if args.action == "new":
+        if not (args.target and args.title):
+            env.error(USAGE, "new needs a number and a title")
+            return env
+        existing = _problem_dirs(root, args.target)
+        if existing and not args.force:
+            env.error(
+                USAGE,
+                f"problem {args.target} already exists at {existing[0]}, add --force to rewrite it",
+            )
+            return env
+        if len(existing) > 1:
+            env.error(USAGE, f"problem {args.target} matches {len(existing)} folders under {root}")
+            return env
+        template = files("meltify.data").joinpath(f"brief_{conf.get('template', 'en')}.md")
+        if not template.is_file():
+            env.error(USAGE, f"no brief template {conf.get('template')!r}, use en or ko")
+            return env
+        d = existing[0] if existing else root / f"{int(args.target):02d}-{_slug(args.title)}"
+        _new(d, args.target, args.title, args.source, template.read_text("utf-8"))
+        env.artifact(str(d / "NOTES.md"), "notes")
+        env.summary = f"{'rewrote' if existing else 'created'} {d}"
+        return env
+
+    if args.action == "set":
+        found = _problem_dirs(root, args.target) if args.target else []
+        if len(found) != 1:
+            what = "no problem" if not found else f"{len(found)} folders for problem"
+            env.error(USAGE, f"{what} {args.target} under {root}")
+            return env
+        status_path = found[0] / "status.json"
+        status = json.loads(status_path.read_text("utf-8"))
+        for key in ("state", "points", "answer"):
+            if (value := getattr(args, key)) is not None:
+                status[key] = value
+        status_path.write_text(json.dumps(status, ensure_ascii=False), "utf-8")
+
+    dirs = sorted(p for p in root.iterdir() if p.is_dir()) if root.exists() else []
+    complete = []
+    for d in dirs:
+        missing = [n for n in ("status.json", "NOTES.md") if not (d / n).is_file()]
+        if missing:
+            env.warnings.append(f"skipped {d}: no {' and no '.join(missing)}")
+        else:
+            complete.append(d)
+    env.results = _board(complete, time.time())
+    env.columns = BOARD_COLUMNS
+    env.summary = f"{len(env.results)} problems under {root}"
+    return env

meltify/commands/check.py

@@ -0,0 +1,305 @@
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import re
+import unicodedata
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from meltify import __version__
+from meltify.evidence import USAGE, Envelope, Src, finding
+
+NAME = "check"
+HELP = "check an answer file or string against its required output format before you hand it in"
+COLUMNS = ["rule", "message", "value", "cite"]
+VIOLATION = "violation"
+
+
+def add_arguments(p: argparse.ArgumentParser) -> None:
+    p.add_argument("file", nargs="?", type=Path, help="JSON, JSONL or CSV answer file")
+    p.add_argument("--text", help="check one string instead of a file")
+    p.add_argument("--schema", type=Path, help="JSON Schema file")
+    p.add_argument("--count", type=int, help="exact number of top-level items")
+    p.add_argument("--unique", action="append", help="key that can't repeat across items")
+    p.add_argument(
+        "--field",
+        action="append",
+        default=[],
+        metavar="PATH",
+        help="JSONPath the text rules apply to, like $[*].id (repeatable)",
+    )
+    p.add_argument("--pattern", help="regex the whole value must match")
+    p.add_argument("--words", type=int, help="exact word count")
+    p.add_argument("--max-words", type=int, help="maximum word count")
+    p.add_argument("--upper", action="store_true", help="no lowercase letters")
+    p.add_argument("--digits", action="store_true", help="ASCII digits only")
+    p.add_argument("--include", action="append", help="phrase the value must contain")
+    p.add_argument("--no-hygiene", action="store_true", help="skip whitespace and width checks")
+
+
+@dataclass(frozen=True)
+class Rule:
+    """Text format rules for the values at one JSONPath"""
+
+    path: str = "$"
+    pattern: str | None = None
+    words: int | None = None
+    max_words: int | None = None
+    upper: bool = False
+    digits: bool = False
+    include: tuple[str, ...] = ()
+
+    @classmethod
+    def from_config(cls, raw: dict[str, Any]) -> Rule:
+        return cls(
+            path=raw.get("path", "$"),
+            pattern=raw.get("pattern"),
+            words=raw.get("words"),
+            max_words=raw.get("max_words"),
+            upper=bool(raw.get("upper", False)),
+            digits=bool(raw.get("digits", False)),
+            include=tuple(raw.get("must_include", raw.get("include", ()))),
+        )
+
+    def problems(self, value: Any) -> Iterator[tuple[str, str]]:
+        if not isinstance(value, str):
+            yield "type", f"expected a string, got {type(value).__name__}"
+            return
+        # \d also matches full-width digits, so spell out the ASCII class
+        if self.digits and not re.fullmatch(r"[0-9]+", value):
+            yield "digits", "ASCII digits only"
+        if self.upper and value != value.upper():
+            yield "upper", "contains lowercase letters"
+        if self.pattern and not re.fullmatch(self.pattern, value):
+            yield "pattern", f"doesn't fully match {self.pattern}"
+        n = len(value.split())
+        if self.words is not None and n != self.words:
+            yield "words", f"{n} words, expected {self.words}"
+        if self.max_words is not None and n > self.max_words:
+            yield "max_words", f"{n} words, at most {self.max_words}"
+        for phrase in self.include:
+            if phrase not in value:
+                yield "include", f"missing {phrase!r}"
+
+
+def hygiene(value: str) -> Iterator[tuple[str, str]]:
+    if value != value.strip():
+        yield "hygiene", "leading or trailing whitespace"
+    if "  " in value:
+        yield "hygiene", "double space"
+    if unicodedata.normalize("NFKC", value) != value:
+        yield "hygiene", "full-width or compatibility characters"
+
+
+PATH_TOKEN = re.compile(r"\.([^.\[\]]+)|\[(\*|-?\d+)\]")
+
+
+def path_tokens(path: str) -> list[tuple[str, str]]:
+    """Minimal JSONPath: dot keys, [n] and [*]"""
+    body = path.removeprefix("$")
+    found = list(PATH_TOKEN.finditer(body))
+    # Reject syntax like filters, which would otherwise select nothing and look like missing data
+    if not path.startswith("$") or "".join(m.group(0) for m in found) != body:
+        raise ValueError(path)
+    return [(m.group(1) or "", m.group(2) or "") for m in found]
+
+
+def select(data: Any, path: str) -> Iterator[tuple[str, Any]]:
+    nodes: list[tuple[str, Any]] = [("$", data)]
+    for key, index in path_tokens(path):
+        nxt = []
+        for p, node in nodes:
+            if key and isinstance(node, dict) and key in node:
+                nxt.append((f"{p}.{key}", node[key]))
+            elif index == "*" and isinstance(node, list):
+                nxt.extend((f"{p}[{i}]", v) for i, v in enumerate(node))
+            elif index == "*" and isinstance(node, dict):
+                nxt.extend((f"{p}.{k}", v) for k, v in node.items())
+            elif (
+                index
+                and index != "*"
+                and isinstance(node, list)
+                and -len(node) <= int(index) < len(node)
+            ):
+                nxt.append((f"{p}[{index}]", node[int(index)]))
+        nodes = nxt
+    return iter(nodes)
+
+
+def strings(data: Any, path: str = "$") -> Iterator[tuple[str, str]]:
+    if isinstance(data, str):
+        yield path, data
+    elif isinstance(data, dict):
+        for k, v in data.items():
+            yield from strings(v, f"{path}.{k}")
+    elif isinstance(data, list):
+        for i, v in enumerate(data):
+            yield from strings(v, f"{path}[{i}]")
+
+
+def load(path: Path) -> Any:
+    suffix = path.suffix.lower()
+    text = path.read_text("utf-8-sig")
+    if suffix == ".csv":
+        return list(csv.DictReader(text.splitlines()))
+    if suffix == ".jsonl":
+        return [json.loads(line) for line in text.splitlines() if line.strip()]
+    return json.loads(text)
+
+
+@dataclass
+class Checker:
+    source: str
+    rules: list[Rule] = field(default_factory=list)
+    schema: dict[str, Any] | None = None
+    count: int | None = None
+    unique: list[str] = field(default_factory=list)
+    check_hygiene: bool = True
+
+    def run(self, data: Any) -> list[dict[str, Any]]:
+        found: list[dict[str, Any]] = []
+
+        def add(rule: str, message: str, jpath: str, value: Any = None) -> None:
+            found.append(
+                finding(Src(self.source, jpath=jpath), rule=rule, message=message, value=value)
+            )
+
+        if self.schema is not None:
+            from jsonschema import Draft202012Validator
+
+            for e in Draft202012Validator(self.schema).iter_errors(data):
+                add("schema", e.message, e.json_path, e.instance if _scalar(e.instance) else None)
+        if not isinstance(data, list):
+            # count and unique describe a list of items, so any other shape fails them
+            got = type(data).__name__
+            if self.count is not None:
+                add("count", f"expected a list of items, got {got}", "$")
+            if self.unique:
+                add("unique", f"expected a list of items, got {got}", "$")
+        elif self.count is not None and len(data) != self.count:
+            add("count", f"{len(data)} items, expected {self.count}", "$", len(data))
+        for key in self.unique:
+            # Include the type in the key, since 1 == True in Python
+            seen: dict[tuple[type, Any], int] = {}
+            for i, item in enumerate(data if isinstance(data, list) else []):
+                if not isinstance(item, dict) or key not in item or not _scalar(item[key]):
+                    continue
+                v = item[key]
+                if (type(v), v) in seen:
+                    add("unique", f"{key} repeats item {seen[type(v), v]}", f"$[{i}].{key}", v)
+                    continue
+                seen[type(v), v] = i
+        if self.check_hygiene:
+            for jpath, s in strings(data):
+                for rule, message in hygiene(s):
+                    add(rule, message, jpath, s)
+        for r in self.rules:
+            matched = list(select(data, r.path))
+            if not matched:
+                add("path", f"nothing at {r.path}", r.path)
+            for jpath, value in matched:
+                for rule, message in r.problems(value):
+                    add(rule, message, jpath, value if _scalar(value) else None)
+        return found
+
+
+def _scalar(v: Any) -> bool:
+    return isinstance(v, str | int | float | bool) or v is None
+
+
+def _has_text_rule(args: argparse.Namespace) -> bool:
+    return any(
+        [
+            args.pattern,
+            args.words is not None,
+            args.max_words is not None,
+            args.upper,
+            args.digits,
+            args.include,
+        ]
+    )
+
+
+def _cli_rule(args: argparse.Namespace, path: str) -> Rule:
+    return Rule(
+        path=path,
+        pattern=args.pattern,
+        words=args.words,
+        max_words=args.max_words,
+        upper=args.upper,
+        digits=args.digits,
+        include=tuple(args.include or ()),
+    )
+
+
+def run(args: argparse.Namespace, settings: dict[str, Any]) -> Envelope:
+    env = Envelope(command=NAME, version=__version__)
+    conf = settings.get("check", {})
+    if (args.file is None) == (args.text is None):
+        env.error(USAGE, "give exactly one of FILE or --text")
+        return env
+
+    hygiene_on = conf.get("hygiene", True) and not args.no_hygiene
+    if args.field and not _has_text_rule(args):
+        env.error(USAGE, "--field needs a text rule, like --pattern or --upper")
+        return env
+    if args.text is not None:
+        ignored = [
+            flag
+            for flag, value in (
+                ("--schema", args.schema),
+                ("--count", args.count),
+                ("--unique", args.unique),
+                ("--field", args.field),
+            )
+            if value not in (None, [])
+        ]
+        if ignored:
+            env.error(USAGE, f"{', '.join(ignored)} apply to a FILE, not --text")
+            return env
+        # Project rules describe an answer file, so a single string uses only its own flags
+        rules = [_cli_rule(args, "$")] if _has_text_rule(args) else []
+    else:
+        if _has_text_rule(args) and not args.field:
+            env.error(USAGE, "text rules on a file need --field, like $[*].id or $.q2")
+            return env
+        rules = [Rule.from_config(r) for r in conf.get("field", [])]
+        if _has_text_rule(args):
+            rules += [_cli_rule(args, p) for p in args.field]
+    for r in rules:
+        try:
+            path_tokens(r.path)
+        except ValueError:
+            env.error(USAGE, f"unsupported path {r.path}, use $ with .key, [n], [-1] and [*]")
+            return env
+        try:
+            re.compile(r.pattern or "")
+        except re.error as e:
+            env.error(USAGE, f"bad pattern {r.pattern!r}: {e}")
+            return env
+    if args.text is not None:
+        checker = Checker(source="<text>", rules=rules, check_hygiene=hygiene_on)
+        data: Any = args.text
+    else:
+        schema_path = args.schema or (Path(conf["schema"]) if conf.get("schema") else None)
+        checker = Checker(
+            source=str(args.file),
+            rules=rules,
+            schema=json.loads(schema_path.read_text("utf-8")) if schema_path else None,
+            count=args.count if args.count is not None else conf.get("count"),
+            unique=args.unique or list(conf.get("unique", [])),
+            check_hygiene=hygiene_on,
+        )
+        env.inputs.append({"path": str(args.file)})
+        data = load(args.file)
+    source = checker.source
+    env.results = checker.run(data)
+    if env.results:
+        env.error(VIOLATION, f"{len(env.results)} violations in {source}")
+    else:
+        env.summary = f"no violations in {source}"
+    return env