tdcv2 0.1.2__py3-none-any.whl

tdcv2/__init__.py

@@ -0,0 +1,6 @@
+"""TDC — The Data Constructor. Deterministic test data from a declarative config."""
+
+from .errors import Diagnostic, Severity, TdcError
+from .tdc import TDC, Row, SeedInfo
+
+__all__ = ["TDC", "Diagnostic", "Row", "SeedInfo", "Severity", "TdcError"]

tdcv2/cli/__init__.py

@@ -0,0 +1,5 @@
+"""The ``tdcv2`` command line.
+
+Nothing is imported here on purpose: ``python -m tdcv2.cli.main`` would otherwise warn about the
+module being loaded twice, and the ordinary library import has no reason to pull the CLI in.
+"""

tdcv2/cli/__main__.py

@@ -0,0 +1,5 @@
+"""``python -m tdcv2.cli`` — the same entry point as the installed ``tdcv2`` command."""
+
+from .main import cli
+
+cli()

tdcv2/cli/args.py

@@ -0,0 +1,149 @@
+"""Command-line arguments, parsed by hand.
+
+`argparse` was the obvious choice and the wrong one. The TypeScript CLI is the interface users
+already know, and matching it exactly matters more than saving fifty lines: `--engine 2` and
+`--engine=2` both work there, an unknown flag is exit code 2 with a specific message, and a
+positional argument may follow flags. Getting argparse to agree on all of that costs more than
+writing the loop, and leaves the agreement undocumented.
+
+Kept apart from `main` so the parsing can be tested without a filesystem or a run.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+class UsageError(ValueError):
+    """A command line that cannot be obeyed. Reported, then exit code 2."""
+
+
+@dataclass(frozen=True, slots=True)
+class Options:
+    """What the command line asked for, before anything has been read or run."""
+
+    input: str | None = None
+    output: str | None = None
+    seed: str | None = None
+    count: int | None = None
+    locale: str | None = None
+    data_paths: list[str] = field(default_factory=list)
+    jobs: int | None = None
+    mode: str | None = None
+    engine: int | None = None
+    help: bool = False
+    version: bool = False
+
+
+#: Flags that take a value, in both the `--flag value` and `--flag=value` spellings.
+_VALUED = (
+    "--output",
+    "--seed",
+    "--count",
+    "--locale",
+    "--data-path",
+    "--jobs",
+    "--mode",
+    "--engine",
+)
+
+
+def parse(argv: list[str]) -> Options:
+    """The generate command's arguments. Raises `UsageError` with the message to print."""
+    out: dict[str, object] = {"data_paths": []}
+    i = 0
+    while i < len(argv):
+        arg = argv[i]
+
+        name, value = _split(arg)
+        if name in _VALUED:
+            if value is None:
+                i += 1
+                if i >= len(argv) or argv[i] == "":
+                    raise UsageError(f"missing value for {name}")
+                value = argv[i]
+            _store(out, name, value)
+            i += 1
+            continue
+
+        if arg in ("-o", "--output"):
+            i += 1
+            if i >= len(argv):
+                raise UsageError("missing value for --output")
+            out["output"] = argv[i]
+        elif arg in ("-h", "--help"):
+            out["help"] = True
+        elif arg in ("-v", "--version"):
+            out["version"] = True
+        elif arg == "--stream":
+            # The legacy spelling of --engine 2, kept because configs and scripts still say it.
+            out["engine"] = 2
+        elif arg == "--disk":
+            out["mode"] = "disk"
+        elif arg.startswith("-"):
+            raise UsageError(f"unknown option: {arg}")
+        elif out.get("input") is None:
+            out["input"] = arg
+        else:
+            raise UsageError(f"unexpected positional argument: {arg}")
+        i += 1
+
+    return Options(**out)  # type: ignore[arg-type]
+
+
+def _split(arg: str) -> tuple[str, str | None]:
+    """`--flag=value` into its parts; anything else unchanged with no value."""
+    if arg.startswith("--") and "=" in arg:
+        name, _, value = arg.partition("=")
+        if value == "":
+            raise UsageError(f"missing value for {name}")
+        return name, value
+    return arg, None
+
+
+def _store(out: dict[str, object], name: str, value: str) -> None:
+    if name == "--output":
+        out["output"] = value
+    elif name == "--seed":
+        out["seed"] = value
+    elif name == "--locale":
+        out["locale"] = value
+    elif name == "--data-path":
+        paths = out["data_paths"]
+        assert isinstance(paths, list)
+        paths.append(value)
+    elif name == "--count":
+        out["count"] = _non_negative(value, "--count")
+    elif name == "--jobs":
+        out["jobs"] = _positive(value, "--jobs")
+    elif name == "--engine":
+        if value not in ("1", "2", "3"):
+            raise UsageError(
+                f'invalid --engine "{value}" — expected 1 (in-memory), 2 (streaming), '
+                "or 3 (exact-on-disk)"
+            )
+        out["engine"] = int(value)
+    elif name == "--mode":
+        if value not in ("memory", "disk"):
+            raise UsageError(f'invalid --mode "{value}" — expected "memory" or "disk"')
+        out["mode"] = value
+
+
+def _non_negative(value: str, flag: str) -> int:
+    try:
+        parsed = int(value)
+    except ValueError:
+        raise UsageError(f'invalid {flag} "{value}" — expected a non-negative integer') from None
+    if parsed < 0:
+        raise UsageError(f'invalid {flag} "{value}" — expected a non-negative integer')
+    return parsed
+
+
+def _positive(value: str, flag: str) -> int:
+    try:
+        parsed = int(value)
+    except ValueError:
+        raise UsageError(f'invalid {flag} "{value}" — expected a positive integer') from None
+    if parsed < 1:
+        raise UsageError(f'invalid {flag} "{value}" — expected a positive integer')
+    return parsed

tdcv2/cli/init.py

@@ -0,0 +1,206 @@
+"""``tdcv2 init`` — create a config file by asking, rather than by making anyone hand-write JSON.
+
+People want to generate data, not learn a config format. In a real terminal this runs a short
+wizard — where the config should live, where downloaded packs go, which locale — and writes the
+file. With no terminal, in a script or in CI, it takes the answers from flags instead, so it stays
+scriptable and testable.
+
+The decisions are pure functions; the wizard is a thin shell over them. That is what the tests
+exercise, because a prompt is hard to test and a decision is not.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+from ..packs import project_config
+
+USAGE = """Usage: tdcv2 init [options]
+
+  -g, --global          Write the per-user config instead of a project one
+  -y, --yes             Take the defaults, ask nothing
+  -f, --force           Overwrite an existing config
+  --locale <loc>        Default locale for the config (default: en)
+  --data-path <dir>     Folder for downloaded packs
+"""
+
+
+class InitError(ValueError):
+    """A command line `init` cannot obey."""
+
+
+@dataclass(frozen=True, slots=True)
+class Plan:
+    """Everything decided, nothing written yet."""
+
+    path: Path
+    pack_store: Path
+    locale: str
+    is_global: bool
+
+
+def config_target(is_global: bool, cwd: Path) -> Path:
+    """Where the config goes: the project's own folder, or the per-user location."""
+    if not is_global:
+        return cwd.resolve() / project_config.CONFIG_NAME
+    target = project_config.global_config_path()
+    if target is None:  # pragma: no cover — every supported platform has one
+        raise InitError("no global config location on this platform — use a project config")
+    return target
+
+
+def default_pack_store(is_global: bool, config_path: Path, cwd: Path) -> Path:
+    """A project keeps packs beside its config; the global config keeps them next to itself."""
+    return config_path.parent / "packs" if is_global else cwd.resolve() / "tdcv2-packs"
+
+
+def config_content(plan: Plan) -> str:
+    """The file's JSON.
+
+    The store is written as ``packStore``, not as a ``dataPaths`` entry: it is where ``pack add``
+    downloads bundles, and it is deliberately not a scan root on its own — each installed bundle
+    registers its own ``packs`` folder, so that addresses stay ``en.person.lastName`` rather than
+    ``en.packs.en.person.lastName``. A project config stores the path relative, so the file can be
+    checked into git and still work on another machine; a global config is machine-specific by
+    nature and stores it absolute.
+    """
+    store = (
+        str(plan.pack_store) if plan.is_global else _relative_to(plan.path.parent, plan.pack_store)
+    )
+    return (
+        json.dumps({"packStore": store, "locale": plan.locale}, indent=2, ensure_ascii=False) + "\n"
+    )
+
+
+def _relative_to(base: Path, target: Path) -> str:
+    absolute, root = target.resolve(), base.resolve()
+    if absolute == root:
+        return "."
+    try:
+        return "./" + absolute.relative_to(root).as_posix()
+    except ValueError:
+        return str(absolute)
+
+
+def write_config(plan: Plan, force: bool) -> None:
+    """Write it, and create the pack folder so ``pack add`` has somewhere to go."""
+    if plan.path.exists() and not force:
+        raise InitError(
+            f'config already exists at "{plan.path}" — pass --force to overwrite, '
+            "or edit it directly"
+        )
+    plan.path.parent.mkdir(parents=True, exist_ok=True)
+    plan.path.write_text(config_content(plan), encoding="utf-8")
+    plan.pack_store.mkdir(parents=True, exist_ok=True)
+
+
+@dataclass(frozen=True, slots=True)
+class Flags:
+    is_global: bool = False
+    force: bool = False
+    yes: bool = False
+    locale: str | None = None
+    pack_store: str | None = None
+
+
+def parse_flags(argv: list[str]) -> Flags:
+    out: dict[str, object] = {}
+    i = 0
+    while i < len(argv):
+        arg = argv[i]
+        if arg in ("-g", "--global"):
+            out["is_global"] = True
+        elif arg in ("-f", "--force"):
+            out["force"] = True
+        elif arg in ("-y", "--yes"):
+            out["yes"] = True
+        elif arg.startswith("--locale="):
+            out["locale"] = arg[len("--locale=") :]
+        elif arg.startswith("--data-path="):
+            out["pack_store"] = arg[len("--data-path=") :]
+        elif arg in ("--locale", "--data-path"):
+            i += 1
+            if i >= len(argv):
+                raise InitError(f"missing value for {arg}")
+            out["locale" if arg == "--locale" else "pack_store"] = argv[i]
+        else:
+            raise InitError(f"unknown option for init: {arg}")
+        i += 1
+    return Flags(**out)  # type: ignore[arg-type]
+
+
+def plan_from_flags(flags: Flags, cwd: Path) -> Plan:
+    path = config_target(flags.is_global, cwd)
+    store = (
+        (cwd / flags.pack_store).resolve()
+        if flags.pack_store
+        else default_pack_store(flags.is_global, path, cwd)
+    )
+    return Plan(path, store, flags.locale or "en", flags.is_global)
+
+
+def run_init(argv: list[str], cwd: Path | None = None) -> int:
+    here = (cwd or Path.cwd()).resolve()
+
+    if any(a in ("-h", "--help") for a in argv):
+        sys.stdout.write(USAGE)
+        return 0
+
+    try:
+        flags = parse_flags(argv)
+    except InitError as e:
+        sys.stderr.write(f"tdcv2: {e}\nRun `tdcv2 init --help` for usage.\n")
+        return 2
+
+    interactive = sys.stdin.isatty() and sys.stdout.isatty() and not flags.yes
+    try:
+        plan = _ask(flags, here) if interactive else plan_from_flags(flags, here)
+    except (EOFError, KeyboardInterrupt):
+        sys.stderr.write("\ntdcv2: cancelled\n")
+        return 1
+    except InitError as e:
+        sys.stderr.write(f"tdcv2: {e}\n")
+        return 2
+
+    try:
+        write_config(plan, flags.force)
+    except (InitError, OSError) as e:
+        sys.stderr.write(f"tdcv2: {e}\n")
+        return 2
+
+    sys.stdout.write(
+        f"Wrote {'global' if plan.is_global else 'project'} config: {plan.path}\n"
+        f"  data packs → {plan.pack_store}\n"
+        f"  locale     → {plan.locale}\n"
+        f"\nNext: run `tdcv2 pack` to download data packs into that folder.\n"
+    )
+    return 0
+
+
+def _ask(flags: Flags, cwd: Path) -> Plan:
+    """The wizard. Plain `input`, because one dependency for three questions is not worth it."""
+    is_global = flags.is_global
+    if not flags.pack_store and not flags.is_global:
+        answer = input(
+            "Where should this config live?\n"
+            "  1) This project — a tdcv2.config.json here, check it into git\n"
+            "  2) Global — all your projects, in your home folder\n"
+            "Choice [1]: "
+        ).strip()
+        is_global = answer == "2"
+
+    path = config_target(is_global, cwd)
+    suggested = (
+        (cwd / flags.pack_store).resolve()
+        if flags.pack_store
+        else default_pack_store(is_global, path, cwd)
+    )
+
+    typed = input(f"Folder for downloaded data packs [{suggested}]: ").strip()
+    store = (cwd / typed).resolve() if typed else suggested
+
+    locale = input(f"Default locale [{flags.locale or 'en'}]: ").strip()
+    return Plan(path, store, locale or flags.locale or "en", is_global)

tdcv2/cli/main.py

@@ -0,0 +1,299 @@
+"""``tdcv2`` — the command line.
+
+The library is the recommended way to embed TDC; this exists so that a `.tdc` file can be run
+without writing a program around it, and so that a Python user never needs another language's
+toolchain to do it. The surface deliberately matches the TypeScript CLI flag for flag: the same
+config run through either must behave the same way, including its exit codes.
+
+Three commands beyond generating:
+
+* ``init``  — write a ``tdcv2.config.json``, by asking rather than by making anyone hand-write JSON
+* ``pack``  — list, install and remove data packs from the shared registry
+* ``check`` — validate a config and say nothing when it is fine
+
+Exit codes: 0 fine, 1 the run failed (an invalid config, a refused preflight), 2 the command line
+itself was wrong.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from ..errors import Diagnostic, Severity, TdcError, format_diagnostic, format_diagnostics
+from .args import Options, UsageError, parse
+
+
+def _version() -> str:
+    """The installed distribution's version, not a second copy of it.
+
+    A hand-written constant here is a number that agrees with itself and with
+    nothing else: bumping ``pyproject.toml`` for a release left ``tdcv2
+    --version`` reporting the old one, silently. The TypeScript package had
+    exactly this bug and it went unnoticed through a release. The fallback
+    covers running from a source tree that was never installed, where there is
+    no distribution to ask.
+    """
+    from importlib.metadata import PackageNotFoundError, version
+
+    try:
+        return version("tdcv2")
+    except PackageNotFoundError:
+        return "0+unknown"
+
+
+VERSION = _version()
+
+HELP = """tdcv2 — The Data Constructor
+
+Usage:
+  tdcv2 <input.tdc> [options]       Generate data from a config
+  tdcv2 init [--global]             Set up a config (asks where; --yes for defaults)
+  tdcv2 pack [list|add|remove <id>] Install / remove data packs (list with no args)
+  tdcv2 check <input.tdc>           Validate a config without generating anything
+  tdcv2 format [-w] <file.tdc>      Pretty-print a config (-w writes it in place)
+
+Options:
+  -o, --output <path>      Write generated content to <path> (default: stdout)
+  --seed <seed>            Override the seed declared in <env>
+  --count <n>              Override the count declared in <env>
+  --locale <loc>           Override the default locale (default: en)
+  --data-path <dir>        Add a data folder for @data/... sources (repeatable)
+  --jobs <n>               Split the run across n processes. Same output either
+                           way — a pure speed knob. Needs -o and the streaming
+                           engine; ignored where the run cannot be split
+  --mode <memory|disk>     Advanced. disk (default): bounded memory, scales to
+                           any size — TDC picks the streaming or exact engine
+                           automatically from the config. memory: the small,
+                           in-RAM engine (an escape hatch; does not scale)
+  --disk                   Shortcut for --mode disk (already the default)
+  --engine <1|2|3>         Advanced: force a specific engine
+  --stream                 Legacy alias for --engine 2
+  -h, --help               Show this message
+  -v, --version            Show version and exit
+
+Data paths also come from tdcv2.config.json (nearest one up from the current
+directory) and the global config — { "dataPaths": [...], "locale": ".." }.
+Order of priority: --data-path > project config > global config > bundled packs.
+
+See https://github.com/NickLiapin/tdcv2 for the DSL reference.
+"""
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Run the CLI. Returns the exit code rather than calling `exit`, so tests can drive it."""
+    args = list(sys.argv[1:] if argv is None else argv)
+
+    if args and args[0] == "init":
+        from .init import run_init
+
+        return run_init(args[1:])
+    if args and args[0] == "pack":
+        from .pack import run_pack
+
+        return run_pack(args[1:])
+    if args and args[0] == "check":
+        return _check(args[1:])
+    if args and args[0] == "format":
+        return _format(args[1:])
+
+    try:
+        options = parse(args)
+    except UsageError as e:
+        _fail(str(e), usage=True)
+        return 2
+
+    if options.help:
+        sys.stdout.write(HELP)
+        return 0
+    if options.version:
+        sys.stdout.write(f"tdcv2 {VERSION}\n")
+        return 0
+    if options.input is None:
+        _fail("input file is required", usage=True)
+        return 2
+
+    return _generate(options)
+
+
+def cli() -> None:
+    """The console-script entry point: run, then exit with the code."""
+    raise SystemExit(main())
+
+
+def _generate(options: Options) -> int:
+    from ..tdc import TDC
+
+    try:
+        data = TDC(
+            options.input,
+            count=options.count,
+            seed=options.seed,
+            locale=options.locale,
+            data_paths=[Path(p) for p in options.data_paths] or None,
+            engine=options.engine if options.engine is not None else _engine_for(options.mode),
+        )
+    except TdcError as e:
+        _report(e.diagnostics, options.input, e.source)
+        return 1
+    except (OSError, ValueError, RuntimeError) as e:
+        # RuntimeError covers the engines' refusals (an infeasible uniq, a
+        # forced streaming engine on a whole-column config) — the reference
+        # prints these as one line, never a stack trace.
+        _fail(str(e))
+        return 1
+
+    _report(data.diagnostics, options.input, data.source)
+
+    # A run with no seed anywhere gets a random one. Print it, or the output cannot be reproduced —
+    # which is the one promise the whole library is built to keep.
+    seed = data.seed_info()
+    if seed.generated:
+        _note(
+            f'no seed specified — using random seed "{seed.seed}". '
+            f'Re-run with --seed "{seed.seed}" to reproduce this exact output.'
+        )
+
+    # Ask what the run will cost before starting it. A config that cannot fit says so in a
+    # millisecond here and takes minutes to say so by thrashing.
+    budget = data.preflight(materialized=options.output is None)
+    if budget is not None:
+        _report_one(budget, options.input, data.source)
+        if budget.severity is Severity.ERROR:
+            return 1
+
+    try:
+        if options.output is not None:
+            data.write_file(options.output, workers=options.jobs)
+        else:
+            # Straight to stdout in one write per record batch rather than per record: a syscall a
+            # row is most of the cost of a large run that is not being written to a file.
+            sys.stdout.write(str(data))
+    except (OSError, ValueError, RuntimeError) as e:
+        # RuntimeError is the engines' refusal channel (an infeasible uniq, a
+        # forced streaming engine on a whole-column config): one line, exit 1,
+        # never a stack trace — the reference's behavior.
+        _fail(str(e))
+        return 1
+    return 0
+
+
+def _check(argv: list[str]) -> int:
+    """``tdcv2 check <file>`` — the validator alone, for an editor or a pre-commit hook."""
+    files = [a for a in argv if not a.startswith("-")]
+    if len(argv) != len(files) or len(files) != 1:
+        _fail("usage: tdcv2 check <input.tdc>")
+        return 2
+
+    from ..tdc import TDC
+
+    try:
+        data = TDC(files[0])
+    except TdcError as e:
+        _report(e.diagnostics, files[0], e.source)
+        return 1
+    except (OSError, ValueError) as e:
+        _fail(str(e))
+        return 1
+
+    problems = data.diagnostics
+    _report(problems, files[0], data.source)
+    if not problems:
+        sys.stderr.write(f"tdcv2: {files[0]} is valid\n")
+    return 0
+
+
+def _format(argv: list[str]) -> int:
+    """``tdcv2 format [-w] <file.tdc>`` — pretty-print a config.
+
+    Prints to stdout by default; ``-w`` overwrites the file. A file with a syntax error is
+    reported and left alone: reformatting something that cannot be parsed would be a guess about
+    what the author meant.
+    """
+    write = False
+    files: list[str] = []
+    for arg in argv:
+        if arg in ("-w", "--write"):
+            write = True
+        elif arg in ("-h", "--help"):
+            sys.stdout.write("Usage: tdcv2 format [-w|--write] <file.tdc>\n")
+            return 0
+        elif arg.startswith("-"):
+            _fail(f"format: unknown option: {arg}")
+            return 2
+        else:
+            files.append(arg)
+
+    if len(files) != 1:
+        _fail("format: a .tdc file is required")
+        return 2
+
+    from ..formatter import format_tdc
+
+    path = Path(files[0])
+    try:
+        source = path.read_text(encoding="utf-8")
+    except OSError as e:
+        _fail(f"format: cannot read {files[0]}: {e}")
+        return 1
+
+    # Never format a file we cannot fully parse — report the syntax error instead.
+    from ..parser import facade
+
+    parsed = facade.parse(source)
+    if not parsed.ok:
+        problems = [
+            Diagnostic.error("TDC001", p.message, "", p.line, p.column) for p in parsed.problems
+        ]
+        _report(problems, files[0], source)
+        return 1
+
+    formatted = format_tdc(source)
+    if not write:
+        sys.stdout.write(formatted)
+        return 0
+
+    if formatted != source:
+        # Write beside the file and rename over it: a crash mid-write must not
+        # leave the user's config truncated.
+        tmp = path.with_name(path.name + ".tmp")
+        tmp.write_text(formatted, encoding="utf-8")
+        tmp.replace(path)
+        _note(f"formatted {files[0]}")
+    else:
+        _note(f"{files[0]} is already formatted")
+    return 0
+
+
+def _engine_for(mode: str | None) -> int | None:
+    """``--mode memory`` is the in-memory engine by name; ``disk`` is the default already."""
+    return 1 if mode == "memory" else None
+
+
+def _report(problems: list[Diagnostic], filename: str | None, source: str | None = None) -> None:
+    """Diagnostics to stderr, so they stay out of a piped or redirected run's data."""
+    if not problems:
+        return
+    sys.stderr.write(
+        format_diagnostics(problems, source, filename or "<input>", sys.stderr.isatty()) + "\n"
+    )
+
+
+def _report_one(problem: Diagnostic, filename: str | None, source: str | None = None) -> None:
+    sys.stderr.write(
+        format_diagnostic(problem, source, filename or "<input>", sys.stderr.isatty()) + "\n"
+    )
+
+
+def _fail(message: str, usage: bool = False) -> None:
+    sys.stderr.write(f"tdcv2: {message}\n")
+    if usage:
+        sys.stderr.write("Run `tdcv2 --help` for usage.\n")
+
+
+def _note(message: str) -> None:
+    sys.stderr.write(f"tdcv2: {message}\n")
+
+
+if __name__ == "__main__":
+    cli()