pycli-dsl 0.1.0__py3-none-any.whl

pycli/__init__.py

@@ -0,0 +1,340 @@
+"""pycli: Python-compatible DevOps DSL.
+
+Transpiles .spy files to standard Python with first-class shell execution.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import traceback
+from pathlib import Path
+
+from pycli.highlighter import highlight_python, should_colorize
+from pycli.importer import SpyFinder, SpyLoader, install_import_hook, uninstall_import_hook
+from pycli.lexer import LexerError
+from pycli.parser import ParseError
+from pycli.repl import start_repl
+from pycli.runtime import (
+    BackgroundJob,
+    CommandError,
+    CommandResult,
+    CommandTimeoutError,
+    DynamicObj,
+    ShellOp,
+    async_run,
+    async_run_expanded,
+    cd,
+    env,
+    run,
+    run_bg,
+    run_bg_expanded,
+    run_expanded,
+    shell_quote,
+    wait_all,
+)
+from pycli.transformer import Transformer, TranspilerError, transpile
+
+__version__ = "0.1.0"
+MAX_SOURCE_SIZE_BYTES = 10 * 1024 * 1024
+
+
+def get_max_source_size() -> int:
+    """Return maximum allowed source size in bytes from env or default."""
+    return int(os.environ.get("PYCLI_MAX_SOURCE_SIZE", MAX_SOURCE_SIZE_BYTES))
+
+__all__ = [
+    "transpile",
+    "TranspilerError",
+    "ParseError",
+    "LexerError",
+    "highlight_python",
+    "install_import_hook",
+    "uninstall_import_hook",
+    "SpyFinder",
+    "SpyLoader",
+    "run",
+    "run_expanded",
+    "run_bg",
+    "run_bg_expanded",
+    "wait_all",
+    "async_run",
+    "async_run_expanded",
+    "BackgroundJob",
+    "cd",
+    "env",
+    "shell_quote",
+    "ShellOp",
+    "start_repl",
+    "CommandResult",
+    "CommandError",
+    "CommandTimeoutError",
+    "DynamicObj",
+    "main",
+]
+
+
+def transpile_file(
+    input_path: Path,
+    output_path: Path | None = None,
+    validate: bool = False,
+    unsafe_interpolation: bool = False,
+) -> str:
+    """Transpile a .spy file to Python code, optionally writing to output_path."""
+    resolved = input_path.resolve()
+    if not resolved.is_file():
+        raise FileNotFoundError(f"File not found: {input_path}")
+
+    size = resolved.stat().st_size
+    max_size = get_max_source_size()
+    if size > max_size:
+        raise ValueError(
+            f"File {input_path} exceeds maximum allowed size "
+            f"({size} bytes > {max_size} bytes)"
+        )
+
+    source = resolved.read_text(encoding="utf-8")
+    transformer = Transformer(auto_import=True, unsafe_interpolation=unsafe_interpolation)
+    py_code = transformer.transform(source, validate=validate)
+    if output_path:
+        output_path.write_text(py_code, encoding="utf-8")
+    return py_code
+
+
+def _make_spy_excepthook(script_path: Path, source_map: dict[int, int], original_hook):
+    resolved_path_str = str(script_path.resolve())
+    orig_path_str = str(script_path)
+
+    def _spy_excepthook(exc_type, exc_value, exc_tb):
+        try:
+            te = traceback.TracebackException(exc_type, exc_value, exc_tb)
+            spy_text = script_path.read_text(encoding="utf-8").splitlines()
+            for frame in te.stack:
+                if frame.filename in (resolved_path_str, orig_path_str, "<string>"):
+                    frame.filename = orig_path_str
+                    if frame.lineno in source_map:
+                        spy_line = source_map[frame.lineno]
+                        frame.lineno = spy_line
+                        if 1 <= spy_line <= len(spy_text):
+                            frame._line = spy_text[spy_line - 1]
+            for line in te.format():
+                sys.stderr.write(line)
+        except Exception:
+            original_hook(exc_type, exc_value, exc_tb)
+
+    return _spy_excepthook
+
+
+def run_file(
+    script_path: Path,
+    script_args: list[str] | None = None,
+    validate: bool = False,
+    unsafe_interpolation: bool = False,
+    warn_external: bool = False,
+) -> int:
+    """Transpile and execute a .spy file using CPython with .spy import hook enabled."""
+    resolved_path = script_path.resolve()
+    if not resolved_path.is_file():
+        sys.stderr.write(f"Error: script not found: {script_path}\n")
+        return 1
+
+    size = resolved_path.stat().st_size
+    max_size = get_max_source_size()
+    if size > max_size:
+        sys.stderr.write(
+            f"Error: File {script_path} exceeds maximum allowed size "
+            f"({size} bytes > {max_size} bytes)\n"
+        )
+        return 1
+
+    if warn_external:
+        sys.stderr.write(
+            "Warning: Executing .spy script with full user privileges. pycli does not sandbox execution.\n"
+        )
+
+    install_import_hook(unsafe_interpolation=unsafe_interpolation, validate=validate)
+
+    # Ensure the script's directory is at the beginning of sys.path
+    script_dir = str(resolved_path.parent)
+    original_sys_path = list(sys.path)
+    if script_dir not in sys.path:
+        sys.path.insert(0, script_dir)
+
+    source = resolved_path.read_text(encoding="utf-8")
+    transformer = Transformer(auto_import=True, unsafe_interpolation=unsafe_interpolation)
+    try:
+        py_code = transformer.transform(source, validate=validate)
+    except (TranspilerError, LexerError, ParseError, ValueError) as e:
+        sys.stderr.write(f"Error: {e}\n")
+        return 1
+
+    original_argv = sys.argv
+    sys.argv = [str(script_path)] + (script_args or [])
+
+    global_namespace: dict = {
+        "__name__": "__main__",
+        "__file__": str(resolved_path),
+        "__doc__": None,
+        "cd": cd,
+        "env": env,
+        "wait_all": wait_all,
+        "run": run,
+        "run_expanded": run_expanded,
+        "run_bg": run_bg,
+        "async_run": async_run,
+        "shell_quote": shell_quote,
+        "ShellOp": ShellOp,
+        "CommandResult": CommandResult,
+        "CommandError": CommandError,
+        "CommandTimeoutError": CommandTimeoutError,
+        "DynamicObj": DynamicObj,
+    }
+
+    orig_excepthook = sys.excepthook
+    sys.excepthook = _make_spy_excepthook(resolved_path, transformer.source_map, orig_excepthook)
+
+    try:
+        compiled = compile(py_code, str(script_path), "exec")
+        exec(compiled, global_namespace)
+        return 0
+    except SystemExit as e:
+        return e.code if isinstance(e.code, int) else (0 if e.code is None else 1)
+    except Exception as e:
+        sys.excepthook(*sys.exc_info())
+        return 1
+    finally:
+        sys.excepthook = orig_excepthook
+        sys.argv = original_argv
+        sys.path = original_sys_path
+
+
+def main(argv: list[str] | None = None) -> None:
+    """CLI entrypoint for pycli."""
+    if argv is None:
+        argv = sys.argv[1:]
+
+    # If no arguments provided in an interactive terminal, launch REPL
+    if not argv:
+        if hasattr(sys.stdin, "isatty") and sys.stdin.isatty():
+            start_repl()
+            return
+
+    parser = argparse.ArgumentParser(
+        prog="pycli",
+        description="pycli: Python-compatible DevOps DSL transpiler and runner.",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"pycli {__version__}",
+    )
+
+    subparsers = parser.add_subparsers(dest="subcommand", help="Available subcommands")
+
+    # pycli repl
+    subparsers.add_parser("repl", help="Start the interactive pycli shell")
+
+    # pycli transpile <file.spy> [-o <file.py>] [--validate] [--unsafe-interpolation] [--color / --no-color]
+    transpile_parser = subparsers.add_parser("transpile", help="Transpile .spy to standard .py")
+    transpile_parser.add_argument("file", help="Path to .spy source file")
+    transpile_parser.add_argument("-o", "--output", help="Output .py file path (defaults to stdout)")
+    transpile_parser.add_argument(
+        "--validate",
+        action="store_true",
+        default=False,
+        help="Validate generated Python using ast.parse() before output",
+    )
+    transpile_parser.add_argument(
+        "--unsafe-interpolation",
+        action="store_true",
+        default=False,
+        help="Disable automatic shell_quote() sanitization on interpolations",
+    )
+    transpile_parser.add_argument(
+        "--color",
+        action="store_true",
+        default=False,
+        help="Force colorized output even when stdout is redirected",
+    )
+    transpile_parser.add_argument(
+        "--no-color",
+        action="store_true",
+        default=False,
+        help="Disable colorized output",
+    )
+
+    # pycli run <file.spy> [--validate] [--unsafe-interpolation] [--warn-external] [args...]
+    run_parser = subparsers.add_parser("run", help="Transpile and execute a .spy file")
+    run_parser.add_argument("file", help="Path to .spy file to run")
+    run_parser.add_argument(
+        "--validate",
+        action="store_true",
+        default=False,
+        help="Validate generated Python before executing",
+    )
+    run_parser.add_argument(
+        "--unsafe-interpolation",
+        action="store_true",
+        default=False,
+        help="Disable automatic shell_quote() sanitization on interpolations",
+    )
+    run_parser.add_argument(
+        "--warn-external",
+        action="store_true",
+        default=False,
+        help="Warn about executing untrusted scripts with user privileges",
+    )
+    run_parser.add_argument("args", nargs=argparse.REMAINDER, help="Arguments passed to script")
+
+    # If the first argument is a file ending in .spy or an existing file (not a known command/flag), default to run
+    if argv and not argv[0].startswith("-") and argv[0] not in ("transpile", "run", "repl", "help"):
+        argv = ["run"] + argv
+
+    args = parser.parse_args(argv)
+
+    if args.subcommand == "repl":
+        start_repl()
+
+    elif args.subcommand == "transpile":
+        input_file = Path(args.file)
+        if not input_file.exists():
+            sys.stderr.write(f"Error: file not found: {input_file}\n")
+            sys.exit(1)
+        out_path = Path(args.output) if args.output else None
+        try:
+            py_code = transpile_file(
+                input_file,
+                out_path,
+                validate=args.validate,
+                unsafe_interpolation=args.unsafe_interpolation,
+            )
+        except (ValueError, TranspilerError, LexerError, ParseError) as e:
+            sys.stderr.write(f"Error: {e}\n")
+            sys.exit(1)
+        if not out_path:
+            if should_colorize(force_color=args.color, no_color=args.no_color):
+                sys.stdout.write(highlight_python(py_code))
+            else:
+                sys.stdout.write(py_code)
+
+    elif args.subcommand == "run":
+        input_file = Path(args.file)
+        if not input_file.exists():
+            sys.stderr.write(f"Error: file not found: {input_file}\n")
+            sys.exit(1)
+        exit_code = run_file(
+            input_file,
+            args.args,
+            validate=args.validate,
+            unsafe_interpolation=args.unsafe_interpolation,
+            warn_external=args.warn_external,
+        )
+        sys.exit(exit_code)
+
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

pycli/highlighter.py

@@ -0,0 +1,122 @@
+"""ANSI syntax highlighting for Python code using Python's standard library tokenizer."""
+
+from __future__ import annotations
+
+import builtins
+import io
+import keyword
+import os
+import sys
+import tokenize
+
+RESET = "\033[0m"
+KW = "\033[94;1m"          # Bright Blue bold (Python keywords)
+DEF_NAME = "\033[93;1m"    # Yellow bold (def / class names)
+BUILTIN = "\033[96m"       # Cyan (Python builtins)
+STR = "\033[92m"           # Bright Green (String literals)
+NUM = "\033[93m"           # Yellow (Numeric literals)
+COMMENT = "\033[90;3m"     # Gray italic (Comments)
+OP = "\033[97m"            # Bright White (Operators & punctuation)
+PYCLI = "\033[95;1m"       # Magenta bold (pycli runtime symbols)
+
+BUILTIN_NAMES = set(dir(builtins))
+PYCLI_NAMES = {
+    "run",
+    "run_expanded",
+    "CommandResult",
+    "CommandError",
+    "DynamicObj",
+    "stdout",
+    "stderr",
+    "exit_code",
+    "duration",
+    "command",
+    "json",
+    "to_dict",
+}
+
+
+def enable_windows_ansi() -> None:
+    """Enable virtual terminal processing on Windows consoles if needed."""
+    if os.name == "nt":
+        os.system("")
+
+
+def highlight_python(code: str) -> str:
+    """Apply ANSI syntax highlighting to Python source code.
+
+    Uses Python's standard library `tokenize` module for lossless, dependency-free
+    syntax coloring.
+    """
+    enable_windows_ansi()
+
+    try:
+        tokens = list(tokenize.tokenize(io.BytesIO(code.encode("utf-8")).readline))
+    except Exception:
+        # Fallback if code cannot be tokenized
+        return code
+
+    lines = code.splitlines(keepends=True)
+    cur_line, cur_col = 1, 0
+    result: list[str] = []
+    prev_tok: tokenize.TokenInfo | None = None
+
+    for t in tokens:
+        if t.type in (tokenize.ENCODING, tokenize.ENDMARKER):
+            continue
+
+        # Emit any whitespace/newlines preceding this token
+        while cur_line < t.start[0]:
+            if cur_line - 1 < len(lines):
+                result.append(lines[cur_line - 1][cur_col:])
+            cur_line += 1
+            cur_col = 0
+        if cur_line - 1 < len(lines) and cur_col < t.start[1]:
+            result.append(lines[cur_line - 1][cur_col : t.start[1]])
+            cur_col = t.start[1]
+
+        # Determine token color
+        color: str | None = None
+        if t.type == tokenize.NAME:
+            if keyword.iskeyword(t.string):
+                color = KW
+            elif prev_tok and prev_tok.string in ("def", "class"):
+                color = DEF_NAME
+            elif t.string in PYCLI_NAMES:
+                color = PYCLI
+            elif t.string in BUILTIN_NAMES:
+                color = BUILTIN
+        elif t.type == tokenize.STRING:
+            color = STR
+        elif t.type == tokenize.NUMBER:
+            color = NUM
+        elif t.type == tokenize.COMMENT:
+            color = COMMENT
+        elif t.type == tokenize.OP:
+            color = OP
+
+        if color:
+            result.append(f"{color}{t.string}{RESET}")
+        else:
+            result.append(t.string)
+
+        cur_line, cur_col = t.end
+        if t.type not in (tokenize.NL, tokenize.NEWLINE, tokenize.COMMENT, tokenize.INDENT, tokenize.DEDENT):
+            prev_tok = t
+
+    # Emit any remaining trailing whitespace or content
+    if cur_line - 1 < len(lines):
+        result.append(lines[cur_line - 1][cur_col:])
+        for rem in lines[cur_line:]:
+            result.append(rem)
+
+    return "".join(result)
+
+
+def should_colorize(force_color: bool = False, no_color: bool = False) -> bool:
+    """Determine whether syntax coloring should be applied to stdout."""
+    if no_color or os.environ.get("NO_COLOR"):
+        return False
+    if force_color:
+        return True
+    return hasattr(sys.stdout, "isatty") and sys.stdout.isatty()

pycli/importer.py

@@ -0,0 +1,214 @@
+"""Import hook system for pycli (.spy) files.
+
+Allows standard Python `import` statements to transparently discover,
+transpile, and execute .spy files as modules and packages.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import importlib.abc
+import importlib.machinery
+import importlib.util
+import marshal
+import sys
+from pathlib import Path
+from typing import Any, Sequence
+
+from pycli.transformer import transpile
+
+STDLIB_NAMES = getattr(sys, "stdlib_module_names", frozenset())
+
+
+def _get_cached_code(
+    spy_path: Path,
+    source: str,
+    unsafe_interpolation: bool = False,
+    validate: bool = False,
+) -> Any:
+    """Retrieve compiled bytecode from __pycache__ or compile and cache it."""
+    cache_dir = spy_path.parent / "__pycache__"
+    cache_key_data = f"{source}\0unsafe={unsafe_interpolation}".encode("utf-8")
+    src_hash = hashlib.sha256(cache_key_data).hexdigest()[:16]
+    cache_file = cache_dir / f"{spy_path.stem}.spy-{src_hash}.pyc"
+
+    if cache_file.is_file():
+        try:
+            data = cache_file.read_bytes()
+            return marshal.loads(data[8:])  # skip 8-byte magic header
+        except Exception:
+            pass
+
+    py_code = transpile(source, unsafe_interpolation=unsafe_interpolation, validate=validate)
+    compiled = compile(py_code, str(spy_path), "exec")
+
+    try:
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        cache_file.write_bytes(b"\x00" * 8 + marshal.dumps(compiled))
+    except Exception:
+        # Silently ignore if filesystem is read-only or not writable
+        pass
+
+    return compiled
+
+
+class SpyLoader(importlib.abc.Loader):
+    """Loader that transpiles .spy code to standard Python and executes it."""
+
+    def __init__(
+        self,
+        fullname: str,
+        path: str | Path,
+        unsafe_interpolation: bool = False,
+        validate: bool = False,
+    ) -> None:
+        self.fullname = fullname
+        self.path = str(Path(path).resolve())
+        self.unsafe_interpolation = unsafe_interpolation
+        self.validate = validate
+
+    def create_module(self, spec: importlib.machinery.ModuleSpec):
+        """Use default Python module creation."""
+        return None
+
+    def exec_module(self, module) -> None:
+        """Transpile .spy file and execute inside module.__dict__."""
+        filepath = Path(self.path).resolve()
+        if not filepath.is_file():
+            raise FileNotFoundError(f"Cannot import .spy module from non-existent path: {self.path}")
+
+        source = filepath.read_text(encoding="utf-8")
+        compiled = _get_cached_code(
+            filepath,
+            source,
+            unsafe_interpolation=self.unsafe_interpolation,
+            validate=self.validate,
+        )
+
+        module.__file__ = str(filepath)
+        module.__loader__ = self
+
+        if filepath.name == "__init__.spy":
+            module.__path__ = [str(filepath.parent)]
+            module.__package__ = self.fullname
+        else:
+            module.__package__ = self.fullname.rpartition(".")[0]
+
+        exec(compiled, module.__dict__)
+
+    def get_source(self, fullname: str) -> str | None:
+        """Return the original .spy source text."""
+        filepath = Path(self.path).resolve()
+        if filepath.is_file():
+            return filepath.read_text(encoding="utf-8")
+        return None
+
+
+class SpyFinder(importlib.abc.MetaPathFinder):
+    """MetaPathFinder that locates .spy files and packages along search paths."""
+
+    def __init__(
+        self,
+        unsafe_interpolation: bool = False,
+        validate: bool = False,
+    ) -> None:
+        self.unsafe_interpolation = unsafe_interpolation
+        self.validate = validate
+
+    def find_spec(
+        self,
+        fullname: str,
+        path: Sequence[str] | None = None,
+        target=None,
+    ) -> importlib.machinery.ModuleSpec | None:
+        top_level = fullname.split(".")[0]
+        # Never shadow standard library modules
+        if top_level in STDLIB_NAMES:
+            return None
+
+        # If already imported and in sys.modules, do not interfere
+        if fullname in sys.modules:
+            return None
+
+        subname = fullname.rpartition(".")[-1]
+
+        # Use package search path if provided, otherwise standard sys.path
+        search_dirs = path if path is not None else sys.path
+
+        for search_dir in search_dirs:
+            if not search_dir or not isinstance(search_dir, (str, Path)):
+                continue
+            dir_path = Path(search_dir).resolve()
+            if not dir_path.is_dir():
+                continue
+
+            # 1. Look for single file module: {dir_path}/{subname}.spy
+            module_file = (dir_path / f"{subname}.spy").resolve()
+            # Ensure no path traversal outside search directory
+            if module_file.is_file() and str(module_file).startswith(str(dir_path)):
+                loader = SpyLoader(
+                    fullname,
+                    module_file,
+                    unsafe_interpolation=self.unsafe_interpolation,
+                    validate=self.validate,
+                )
+                return importlib.util.spec_from_loader(
+                    fullname,
+                    loader,
+                    origin=str(module_file),
+                )
+
+            # 2. Look for package: {dir_path}/{subname}/__init__.spy
+            pkg_dir = (dir_path / subname).resolve()
+            init_file = (pkg_dir / "__init__.spy").resolve()
+            if init_file.is_file() and str(init_file).startswith(str(dir_path)):
+                loader = SpyLoader(
+                    fullname,
+                    init_file,
+                    unsafe_interpolation=self.unsafe_interpolation,
+                    validate=self.validate,
+                )
+                spec = importlib.util.spec_from_loader(
+                    fullname,
+                    loader,
+                    origin=str(init_file),
+                    is_package=True,
+                )
+                if spec:
+                    spec.submodule_search_locations = [str(pkg_dir)]
+                return spec
+
+        return None
+
+
+# Singleton finder instance
+_spy_finder: SpyFinder | None = None
+
+
+def install_import_hook(
+    unsafe_interpolation: bool = False,
+    validate: bool = False,
+) -> SpyFinder:
+    """Install the .spy import hook into sys.meta_path if not already present."""
+    global _spy_finder
+    if _spy_finder is None:
+        _spy_finder = SpyFinder(
+            unsafe_interpolation=unsafe_interpolation,
+            validate=validate,
+        )
+    else:
+        _spy_finder.unsafe_interpolation = unsafe_interpolation
+        _spy_finder.validate = validate
+
+    if _spy_finder not in sys.meta_path:
+        # Insert before standard PathFinder so .spy files can be loaded
+        sys.meta_path.insert(0, _spy_finder)
+
+    return _spy_finder
+
+
+def uninstall_import_hook() -> None:
+    """Remove the .spy import hook from sys.meta_path."""
+    global _spy_finder
+    if _spy_finder is not None and _spy_finder in sys.meta_path:
+        sys.meta_path.remove(_spy_finder)