conlanggen 0.4.2__py3-none-any.whl

conlang/__init__.py

@@ -0,0 +1,29 @@
+"""conlang: a partial-spec constructed-language generator."""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _dist_version
+
+from .lint import LintReport, lint
+from .model import Language
+from .pipeline import generate
+from .pipeline.compose import Composer
+from .render.markdown import render_markdown
+from .spec import Spec, dump_spec, load_spec, to_spec
+
+try:
+    __version__ = _dist_version("conlanggen")
+except PackageNotFoundError:  # running from a source tree without an install
+    __version__ = "0.0.0+unknown"
+
+__all__ = [
+    "Composer",
+    "Language",
+    "LintReport",
+    "Spec",
+    "dump_spec",
+    "generate",
+    "lint",
+    "load_spec",
+    "render_markdown",
+    "to_spec",
+]

conlang/bench/__init__.py

@@ -0,0 +1,5 @@
+"""Package marker for the benchmark layer."""
+
+from .kah import BenchReport, KahEntry, bench_kah, parse_kah_lexicon
+
+__all__ = ["BenchReport", "KahEntry", "bench_kah", "parse_kah_lexicon"]

conlang/bench/kah.py

@@ -0,0 +1,138 @@
+"""Kah benchmark adapter.
+
+Kah is hand-made, so this does *not* try to regenerate Kah. It does two things:
+
+1. Parse the real Kwesho dictionary cache into ``KahEntry`` models -- an
+   integration check that the schema can represent real data. Kah data is
+   copyrighted and is never vendored into this repo; point ``--kah`` at a local
+   cache (see the ``kah-translation`` skill).
+2. Produce a plausibility scorecard comparing a generated language against Kah's
+   statistics. The scorecard is reported, never used as a pass/fail gate.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from ..model import Language
+from ..phonology.syllable import is_legal_word, skeleton, tokenize
+from ..sampling.segments import PRESETS
+
+POS = r"(?:a|s|n|i|c|v|adj|adv|prep|pron|num|conj|intj|interj|art|part|aux)\."
+HEADWORD_RE = re.compile(rf"^(?P<hw>\S+)\s{{1,}}(?P<pos>{POS})\s*(?P<gloss>.*)$")
+ARTIFACT_RE = re.compile(r"^(Kah Dictionary|\d+|[A-Z])$")
+
+# The inventory the dictionary is tokenized with is the single curated Kah
+# inventory, imported from the preset rather than restated, so the preset and
+# the adapter cannot drift apart.
+KAH_INVENTORY = PRESETS["kah"]
+
+
+class KahEntry(BaseModel):
+    headword: str
+    pos: str
+    gloss: str
+
+
+class BenchReport(BaseModel):
+    source: str
+    entries_parsed: int
+    valid_models: int
+    validity_rate: float
+    symbol_coverage: float
+    tokenizable_rate: float
+    phonotactic_legality: float
+    kah_syllable_shapes: list[str] = Field(default_factory=list)
+    lang_syllable_shapes: list[str] = Field(default_factory=list)
+    notes: list[str] = Field(default_factory=list)
+
+
+def section_bounds(lines: list[str]) -> tuple[int, int]:
+    marks = [i for i, line in enumerate(lines) if line.strip() == "A"]
+    if len(marks) >= 2:
+        return marks[0], marks[1]
+    return 0, len(lines)
+
+
+def parse_kah_lexicon(path: str | Path) -> list[KahEntry]:
+    lines = Path(path).read_text(encoding="utf-8").splitlines()
+    lo, hi = section_bounds(lines)
+    entries: list[KahEntry] = []
+    for i in range(lo, hi):
+        stripped = lines[i].strip()
+        if ARTIFACT_RE.match(stripped):
+            continue
+        match = HEADWORD_RE.match(stripped)
+        if match:
+            entries.append(
+                KahEntry(
+                    headword=match.group("hw"),
+                    pos=match.group("pos").rstrip("."),
+                    gloss=match.group("gloss").strip(),
+                )
+            )
+    return entries
+
+
+def kah_syllable_shapes(entries: list[KahEntry], limit: int = 64) -> list[str]:
+    shapes: set[str] = set()
+    for entry in entries:
+        tokens = tokenize(entry.headword, KAH_INVENTORY)
+        if tokens is None:
+            continue
+        shapes.add(skeleton(tokens, KAH_INVENTORY))
+    return sorted(shapes)[:limit]
+
+
+def bench_kah(language: Language, path: str | Path) -> BenchReport:
+    entries = parse_kah_lexicon(path)
+    valid = [e for e in entries if e.headword and e.gloss and e.pos and e.headword.isalpha()]
+
+    kah_symbols = set(KAH_INVENTORY.symbols())
+    lang_symbols = set(language.inventory.symbols())
+    coverage = len(kah_symbols & lang_symbols) / len(kah_symbols) if kah_symbols else 0.0
+
+    # Separate the two ways a Kah headword can fail. A headword this language
+    # cannot even tokenize is missing a *sound*; one that tokenizes but will not
+    # syllabify has a forbidden *shape*. Collapsing the two makes the scorecard
+    # unreadable -- a phonotactics improvement looks like a vocabulary one.
+    tokenizable: list[KahEntry] = []
+    for entry in valid:
+        if tokenize(entry.headword, language.inventory) is not None:
+            tokenizable.append(entry)
+
+    legal = sum(
+        1
+        for entry in tokenizable
+        if is_legal_word(entry.headword, language.inventory, language.syllable_templates)
+    )
+    tokenizable_rate = len(tokenizable) / len(valid) if valid else 0.0
+    legality = legal / len(tokenizable) if tokenizable else 0.0
+
+    notes = [
+        f"{len(tokenizable)}/{len(valid)} headwords use only sounds this language has",
+        f"{legal}/{len(tokenizable)} of those are legal under this language's phonotactics",
+        f"overall: {legal}/{len(valid)} ({legal / len(valid) * 100 if valid else 0:.1f}%)",
+        "Kah is hand-made; this is a plausibility scorecard, not a pass/fail gate",
+    ]
+    if PRESETS["kah"].symbols() == language.inventory.symbols():
+        notes.append(
+            "this language uses the Kah preset, so its scorecard is a self-check, "
+            "not an independent measurement"
+        )
+
+    return BenchReport(
+        source=str(path),
+        entries_parsed=len(entries),
+        valid_models=len(valid),
+        validity_rate=len(valid) / len(entries) if entries else 0.0,
+        symbol_coverage=coverage,
+        tokenizable_rate=tokenizable_rate,
+        phonotactic_legality=legality,
+        kah_syllable_shapes=kah_syllable_shapes(valid),
+        lang_syllable_shapes=[t.pattern for t in language.syllable_templates],
+        notes=notes,
+    )

conlang/cli.py

@@ -0,0 +1,430 @@
+"""Command-line interface: a thin view over the library."""
+
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path
+
+from .bench.kah import bench_kah
+from .lint import lint
+from .model import Language
+from .pipeline import generate
+from .pipeline.compose import compose_translation
+from .pipeline.extras import build_extras
+from .pipeline.grow import grow_lexicon
+from .pipeline.port import PortabilityError, PortabilityReport
+from .pipeline.scaffold import new_language
+from .pipeline.validate import validate_translation
+from .render.markdown import render_markdown
+from .soundchange import evolve
+from .spec import Spec, dump_spec, load_spec, to_spec
+
+
+def _user_cache_dir() -> Path:
+    """A conventional per-user cache directory, with no third-party dependency."""
+    xdg = os.environ.get("XDG_CACHE_HOME")
+    if xdg:
+        return Path(xdg) / "conlang"
+    if os.name == "nt":
+        local = os.environ.get("LOCALAPPDATA")
+        if local:
+            return Path(local) / "conlang"
+    return Path.home() / ".cache" / "conlang"
+
+
+# Where `bench` looks for the Kah glossary when nothing is passed. The data is
+# third-party, so it is never bundled: the user provides it via --kah, the
+# CONLANG_KAH_LEXICON env var, or by dropping it at this path.
+DEFAULT_KAH_CACHE = _user_cache_dir() / "kah-lexicon.txt"
+
+
+def _write(path: Path, text: str) -> None:
+    """Write UTF-8 with LF endings so output bytes are identical on every platform."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8", newline="\n") as handle:
+        handle.write(text)
+
+
+def _load_language(path: str) -> Language:
+    return Language.model_validate_json(Path(path).read_text(encoding="utf-8"))
+
+
+def _cmd_generate(args: argparse.Namespace) -> int:
+    spec = load_spec(args.spec)
+    if args.seed is not None:
+        spec = spec.model_copy(update={"seed": args.seed})
+    language = generate(spec)
+    out = Path(args.out) if args.out else Path("out") / f"{language.id}.json"
+    _write(out, language.model_dump_json(indent=2))
+    report = lint(language)
+    print(
+        f"{language.id}: {len(language.lexemes)} lexemes, "
+        f"{len(language.words)} word forms, seed {language.seed}"
+    )
+    print(f"saved {out}")
+    print(f"lint: {report.status} ({len(report.errors)} errors, {len(report.warnings)} warnings)")
+    for issue in report.issues[:20]:
+        print(f"  [{issue.severity}] {issue.rule}: {issue.message}")
+    if args.render:
+        md = out.with_suffix(".md")
+        _write(md, render_markdown(language))
+        print(f"rendered {md}")
+    return 0
+
+
+def _cmd_render(args: argparse.Namespace) -> int:
+    language = _load_language(args.model)
+    text = render_markdown(language)
+    if args.out:
+        _write(Path(args.out), text)
+        print(f"rendered {args.out}")
+    else:
+        print(text)
+    return 0
+
+
+def _cmd_lint(args: argparse.Namespace) -> int:
+    language = _load_language(args.model)
+    report = lint(language)
+    print(f"{language.id}: {report.status}")
+    for issue in report.issues:
+        print(f"  [{issue.severity}] {issue.rule}: {issue.message}")
+    return 0 if report.status == "valid" else 1
+
+
+def _cmd_bench(args: argparse.Namespace) -> int:
+    path = args.kah or os.environ.get("CONLANG_KAH_LEXICON") or str(DEFAULT_KAH_CACHE)
+    if not Path(path).exists():
+        print(f"Kah cache not found: {path}")
+        print("Pass --kah PATH or set CONLANG_KAH_LEXICON.")
+        return 2
+    if args.model:
+        language = _load_language(args.model)
+    elif args.spec:
+        language = generate(load_spec(args.spec))
+    else:
+        language = generate(Spec())
+
+    report = bench_kah(language, path)
+    print(f"Kah benchmark: {report.source}")
+    print(f"  entries parsed: {report.entries_parsed}")
+    print(f"  valid models:   {report.valid_models} ({report.validity_rate:.1%})")
+    print(f"  symbol coverage: {report.symbol_coverage:.1%}")
+    print(f"  usable sounds:   {report.tokenizable_rate:.1%}")
+    print(f"  legal shapes:    {report.phonotactic_legality:.1%}")
+    print(f"  Kah syllable shapes: {' '.join(report.kah_syllable_shapes)}")
+    print(f"  language templates:  {' '.join(report.lang_syllable_shapes)}")
+    for note in report.notes:
+        print(f"  note: {note}")
+    return 0
+
+
+def _cmd_evolve(args: argparse.Namespace) -> int:
+    """Derive a daughter language from a model by ordered sound changes."""
+    parent = _load_language(args.model)
+    try:
+        daughter = evolve(parent, args.rule, name=args.name)
+    except ValueError as error:
+        print(f"bad rule: {error}")
+        return 2
+    out = Path(args.out) if args.out else Path(args.model).with_name(f"{daughter.id}.json")
+    _write(out, daughter.model_dump_json(indent=2))
+    report = lint(daughter)
+    print(f"{daughter.id}: {len(daughter.lexemes)} lexemes, {len(daughter.words)} word forms")
+    print(f"  derived from {daughter.lineage.parent} by {len(daughter.lineage.rules)} rule(s)")
+    print(f"  saved {out}")
+    print(f"lint: {report.status} ({len(report.errors)} errors, {len(report.warnings)} warnings)")
+    return 0
+
+
+def _cmd_accept(args: argparse.Namespace) -> int:
+    """Freeze a generated model into a fully-explicit spec (write-back)."""
+    language = _load_language(args.model)
+    spec = to_spec(language)
+    out = (
+        Path(args.out)
+        if args.out
+        else Path(args.model).with_name(Path(args.model).stem + ".spec.yaml")
+    )
+    _write(out, dump_spec(spec))
+    affix_count = len(spec.affixes or [])
+    print(f"wrote {out}")
+    print(f"  pinned {len(spec.lexicon)} lexemes and {affix_count} affixes")
+    print("  regenerating from this spec reproduces the same language")
+    return 0
+
+
+def _resolve_under(
+    workspace: str, subdir: str, suffix: str, explicit: str | None, what: str
+) -> Path:
+    """Resolve a workspace path: an explicit flag, else the single match under `subdir`."""
+    if explicit:
+        return Path(explicit)
+    candidates = sorted((Path(workspace) / subdir).glob(f"*{suffix}"))
+    if len(candidates) == 1:
+        return candidates[0]
+    if not candidates:
+        raise ValueError(f"no {what} found under {Path(workspace) / subdir}")
+    raise ValueError(f"multiple {what}s under {Path(workspace) / subdir}; pass one explicitly")
+
+
+def _print_port_report(report: PortabilityReport) -> None:
+    axes = ", ".join(f"{name}={'ok' if ok else 'MISMATCH'}" for name, ok in report.axes.items())
+    print(f"portable: {report.portable} ({axes})")
+    if report.missing_tags:
+        print(f"  missing tags: {report.missing_tags}")
+    if report.missing_glosses:
+        print(f"  missing glosses: {len(report.missing_glosses)} (grow these): "
+              f"{report.missing_glosses[:10]}")
+
+
+def _cmd_new(args: argparse.Namespace) -> int:
+    def _split(value: str | None) -> list[str] | None:
+        return value.split(",") if value else None
+
+    orthography = (
+        dict(pair.split("=", 1) for pair in args.orthography) if args.orthography else None
+    )
+    result = new_language(
+        args.name,
+        seed=args.seed,
+        consonants=_split(args.consonants),
+        vowels=_split(args.vowels),
+        syllables=_split(args.syllables),
+        ipa=args.ipa,
+        orthography=orthography,
+        word_order=args.word_order,
+        alignment=args.alignment,
+        workspace=args.workspace,
+        git=not args.no_git,
+    )
+    print(f"{result.language_id}: {result.lexemes} lexemes, {result.word_forms} word forms")
+    print(f"wrote {result.spec_path}")
+    print(f"wrote {result.model_path}")
+    print(f"scaffolded {len(result.files)} files under {args.workspace}")
+    return 0
+
+
+def _cmd_grow(args: argparse.Namespace) -> int:
+    try:
+        spec = _resolve_under(args.workspace, "spec", ".yaml", args.spec, "spec")
+    except ValueError as error:
+        print(error)
+        return 2
+    result = grow_lexicon(spec, args.words, out=args.out)
+    print(f"added {len(result.added)} -> {[(gloss, root) for gloss, root, _ in result.added]}")
+    if result.duplicates:
+        print(f"duplicates skipped: {result.duplicates}")
+    if result.reserved:
+        print(f"reserved pos skipped: {result.reserved}")
+    print(f"{result.language_id}: {result.lexemes} lexemes, {result.word_forms} word forms")
+    print(f"wrote {result.spec_path} and {result.model_path}")
+    return 0
+
+
+def _cmd_translate(args: argparse.Namespace) -> int:
+    if bool(args.glosses) == bool(args.port_from):
+        print("pass exactly one of --glosses or --port-from")
+        return 2
+    try:
+        model = _resolve_under(args.workspace, "out", ".json", args.model, "model")
+    except ValueError as error:
+        print(error)
+        return 2
+    try:
+        result = compose_translation(
+            model,
+            args.out,
+            args.sentences,
+            glosses=args.glosses,
+            port_from=args.port_from,
+            source_model=args.source_model,
+            column=args.column,
+            check=args.check,
+            force=args.force,
+        )
+    except PortabilityError as error:
+        print(f"not portable: {error}")
+        _print_port_report(error.report)
+        return 1
+    if result.report is not None:
+        _print_port_report(result.report)
+    if args.check:
+        print("check only: nothing written")
+        return 0 if result.report and result.report.portable else 1
+    print(f"{result.language_id}: {result.translated}/{result.total} translated -> {result.out}")
+    for problem in result.problems:
+        print(f"  problem: {problem}")
+    return 1 if result.problems else 0
+
+
+def _cmd_validate_translation(args: argparse.Namespace) -> int:
+    try:
+        model = _resolve_under(args.workspace, "out", ".json", args.model, "model")
+    except ValueError as error:
+        print(error)
+        return 2
+    sentences = args.sentences or str(Path(args.workspace) / "the_quiet_morning_sentences.csv")
+    sentences_arg = sentences if Path(sentences).exists() else None
+    report = validate_translation(
+        args.csv,
+        model,
+        column=args.column,
+        rows=args.rows,
+        header=args.header,
+        sentences=sentences_arg,
+        extras=args.extras,
+    )
+    for issue in report.errors[:50]:
+        print(issue)
+    print(
+        f"translated {report.translated}/{report.total} · "
+        f"slots {report.slots} · errors {len(report.errors)}"
+    )
+    return 1 if report.errors else 0
+
+
+def _cmd_extras(args: argparse.Namespace) -> int:
+    try:
+        model = _resolve_under(args.workspace, "out", ".json", args.model, "model")
+    except ValueError as error:
+        print(error)
+        return 2
+    result = build_extras(
+        args.csv, model, column=args.column, out_dir=args.out_dir, deck=args.deck
+    )
+    print(f"gloss CSV: {result['gloss_csv']}")
+    print(f"docx:      {result['docx']}")
+    print(f"deck:      {result['deck']} ({result['cards']} cards)")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="conlang",
+        description="Generate a conlang from a partial spec; sample the rest.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    gen = sub.add_parser("generate", help="Generate a language model from a spec.")
+    gen.add_argument("--spec", required=True, help="Path to a YAML spec.")
+    gen.add_argument("--seed", type=int, default=None, help="Override the spec seed.")
+    gen.add_argument("--out", default=None, help="Output model JSON path.")
+    gen.add_argument("--render", action="store_true", help="Also render Markdown.")
+    gen.set_defaults(func=_cmd_generate)
+
+    ren = sub.add_parser("render", help="Render a model JSON to Markdown.")
+    ren.add_argument("--model", required=True, help="Path to a model JSON.")
+    ren.add_argument("--out", default=None, help="Output Markdown path (default stdout).")
+    ren.set_defaults(func=_cmd_render)
+
+    lin = sub.add_parser("lint", help="Lint a model JSON.")
+    lin.add_argument("--model", required=True, help="Path to a model JSON.")
+    lin.set_defaults(func=_cmd_lint)
+
+    acc = sub.add_parser(
+        "accept",
+        help="Freeze a model into an explicit spec (write-back), so sampled values stick.",
+    )
+    acc.add_argument("--model", required=True, help="Path to a model JSON.")
+    acc.add_argument("--out", default=None, help="Output spec path (default <model>.spec.yaml).")
+    acc.set_defaults(func=_cmd_accept)
+
+    ben = sub.add_parser("bench", help="Score a language against the local Kah cache.")
+    ben.add_argument("--model", default=None, help="Path to a model JSON.")
+    ben.add_argument("--spec", default=None, help="Generate from this spec, then bench.")
+    ben.add_argument("--kah", default=None, help="Path to kah-lexicon.txt.")
+    ben.set_defaults(func=_cmd_bench)
+
+    evo = sub.add_parser(
+        "evolve", help="Derive a daughter language by ordered sound changes."
+    )
+    evo.add_argument("--model", required=True, help="Path to the parent model JSON.")
+    evo.add_argument(
+        "--rule",
+        action="append",
+        required=True,
+        metavar="OLD > NEW",
+        help="A sound change, e.g. 'a > e'. Repeatable; applied in order.",
+    )
+    evo.add_argument("--name", default=None, help="Daughter language id.")
+    evo.add_argument("--out", default=None, help="Output model JSON path.")
+    evo.set_defaults(func=_cmd_evolve)
+
+    new = sub.add_parser("new", help="Scaffold and freeze a new generated language.")
+    new.add_argument("--name", required=True)
+    new.add_argument("--seed", type=int, default=1)
+    new.add_argument("--consonants", default=None, help="comma-separated symbols")
+    new.add_argument("--vowels", default=None, help="comma-separated symbols")
+    new.add_argument("--syllables", default=None, help="comma-separated, e.g. CV,CVC")
+    new.add_argument("--ipa", action="store_true", help="treat the inventory as IPA")
+    new.add_argument(
+        "--orthography", action="append", default=None, metavar="SYM=SPELL",
+        help="orthography override, repeatable (e.g. ʃ=sh)",
+    )
+    new.add_argument(
+        "--word-order", dest="word_order", default=None, choices=["SOV", "SVO", "VSO"],
+        help="pin the word order (default: sampled)",
+    )
+    new.add_argument(
+        "--alignment", default=None, choices=["accusative", "ergative"],
+        help="pin the alignment (default: sampled)",
+    )
+    new.add_argument("--workspace", default=".", help="workspace root (default: .)")
+    new.add_argument("--no-git", action="store_true", help="do not git init the workspace")
+    new.set_defaults(func=_cmd_new)
+
+    grow = sub.add_parser("grow", help="Grow a frozen spec's lexicon, in place.")
+    grow.add_argument("--words", required=True, help="JSON list/map of gloss -> pos")
+    grow.add_argument("--workspace", default=".")
+    grow.add_argument("--spec", default=None, help="default: <workspace>/spec/*.yaml")
+    grow.add_argument("--out", default=None, help="model JSON path")
+    grow.set_defaults(func=_cmd_grow)
+
+    tr = sub.add_parser("translate", help="Compose a translation, or port a sibling's.")
+    tr.add_argument("--workspace", default=".")
+    tr.add_argument("--model", default=None, help="default: <workspace>/out/*.json")
+    tr.add_argument("--glosses", default=None, help="hand-authored gloss-script TSV")
+    tr.add_argument("--port-from", dest="port_from", default=None, help="source _gloss.csv")
+    tr.add_argument("--source-model", dest="source_model", default=None, help="the source model")
+    tr.add_argument("--sentences", required=True, help="source sentences CSV")
+    tr.add_argument("--out", required=True, help="output translation CSV")
+    tr.add_argument("--column", default=None, help="language column (default: the model id)")
+    tr.add_argument("--check", action="store_true", help="report portability, write nothing")
+    tr.add_argument("--force", action="store_true", help="port despite a grammar mismatch")
+    tr.set_defaults(func=_cmd_translate)
+
+    val = sub.add_parser("validate-translation", help="Validate a translation CSV.")
+    val.add_argument("--csv", required=True)
+    val.add_argument("--workspace", default=".")
+    val.add_argument("--model", default=None, help="default: <workspace>/out/*.json")
+    val.add_argument("--column", default=None)
+    val.add_argument("--rows", type=int, default=0, help="expected rows (default: derived)")
+    val.add_argument("--header", default=None, help="expected header, comma-separated")
+    val.add_argument("--sentences", default=None, help="source sentences CSV")
+    val.add_argument(
+        "--extras", action="store_true",
+        help="emit the gloss CSV / docx / deck, then check completeness",
+    )
+    val.set_defaults(func=_cmd_validate_translation)
+
+    ex = sub.add_parser("extras", help="Render a translation's gloss CSV, docx, and deck.")
+    ex.add_argument("--csv", required=True)
+    ex.add_argument("--workspace", default=".")
+    ex.add_argument("--model", default=None, help="default: <workspace>/out/*.json")
+    ex.add_argument("--column", default=None)
+    ex.add_argument("--out-dir", dest="out_dir", default=None)
+    ex.add_argument("--deck", default=None)
+    ex.set_defaults(func=_cmd_extras)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    return int(args.func(args))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())