cixstack 1.0.0rc1__py3-none-any.whl

cixstack/__main__.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+# PYTHON_ARGCOMPLETE_OK
+import argparse
+import json
+import os
+import shlex
+import signal
+import subprocess
+import sys
+import textwrap
+import traceback
+from pathlib import Path
+from typing import TYPE_CHECKING, Protocol
+
+import argcomplete
+import attrs
+import diskcache
+import dotenv
+
+import cixstack.cli.alias
+import cixstack.cli.cache
+import cixstack.cli.cleanup
+import cixstack.cli.config
+import cixstack.cli.fix
+import cixstack.cli.hook
+import cixstack.cli.init
+import cixstack.cli.match_
+import cixstack.cli.run
+import cixstack.cli.start
+import cixstack.cli.status
+import cixstack.cli.watch
+import cixstack.errors
+import cixstack.utils
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, MutableMapping, Sequence
+    from typing import Any
+
+
+ENV_FILE_PATH = Path.cwd() / ".env"
+DEFAULT_COMMANDS = {
+    "run": "Run jobs",
+    "fix": "Run fixes for jobs with a `fix:` defined",
+    "cleanup": "Run cleanups for jobs with a `cleanup:` defined",
+    "status": "Display the status of all jobs",
+    "watch": "Continuously rerun status",
+    "start": "Start a service",
+    "match": "Show files matched by jobs",
+    "init": "Create a default configuration file",
+    "config": "Show configuration options",
+    "cache": "Manage the cache",
+    "hook": "Create git hooks",
+}
+
+
+Context = (
+    cixstack.cli.run.Context
+    | cixstack.cli.match_.Context
+    | cixstack.cli.init.Context
+    | cixstack.cli.config.Context
+    | cixstack.cli.cache.Context
+    | cixstack.cli.hook.Context
+)
+
+
+class Module(Protocol):
+    @staticmethod
+    def parse_args(cache: MutableMapping[bytes, Any], args: Sequence[str]) -> Context: ...
+
+    @staticmethod
+    def main(cache: MutableMapping[bytes, Any], ctx: Context) -> int: ...
+
+
+class Runnable(Protocol):
+    def run(self, cache: MutableMapping[bytes, Any]) -> int: ...
+
+
+class Noop:
+    def run(self, cache: MutableMapping[bytes, Any]) -> int:
+        del cache  # unused
+        return 0
+
+
+@attrs.define
+class CommandProtocol:
+    ctx: Context
+    main: Callable[[MutableMapping[bytes, Any], Context], int]
+
+    def run(self, cache: MutableMapping[bytes, Any]) -> int:
+        return self.main(cache, self.ctx)
+
+    @classmethod
+    def from_module(
+        cls,
+        cache: MutableMapping[bytes, Any],
+        module: Module,
+        args: Sequence[str],
+    ) -> CommandProtocol:
+        return cls(ctx=module.parse_args(cache, args), main=module.main)
+
+
+@attrs.define
+class SubprocessProtocol:
+    argv: tuple[str, ...] = attrs.field(
+        converter=tuple,
+        validator=attrs.validators.deep_iterable(
+            member_validator=attrs.validators.instance_of(str)
+        ),
+    )
+    check: bool
+    shell: bool
+
+    def run(self, cache: MutableMapping[bytes, Any]) -> int:
+        del cache  # unused
+        result = subprocess.run(self.argv, check=self.check, shell=self.shell)
+        return result.returncode
+
+
+def _parse_outer_args(argv: Sequence[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "command",
+        type=str,
+        help="Command to run",
+    ).completer = argcomplete.completers.ChoicesCompleter(DEFAULT_COMMANDS)
+
+    argcomplete.autocomplete(parser)
+
+    return parser.parse_args(argv[:1])
+
+
+def parse_args(cache: MutableMapping[bytes, Any], argv: Sequence[str]) -> Runnable:
+    if "_ARGCOMPLETE" in os.environ:
+        complete_start_point = int(os.environ["_ARGCOMPLETE"])
+        complete_line = shlex.split(os.environ["COMP_LINE"])
+
+        if len(complete_line) <= complete_start_point:
+            _parse_outer_args(argv)  # Trigger argcomplete for the outer parser
+            return Noop()
+
+        os.environ["COMP_LINE"] = shlex.join(complete_line[complete_start_point:])
+        argv = complete_line[complete_start_point:]
+
+    match argv:
+        case ["run", *args]:
+            return CommandProtocol.from_module(cache, cixstack.cli.run.Run, args)
+        case ["fix", *args]:
+            return CommandProtocol.from_module(cache, cixstack.cli.fix.Fix, args)
+        case ["cleanup", *args]:
+            return CommandProtocol.from_module(cache, cixstack.cli.cleanup.Cleanup, args)
+        case ["status", *args]:
+            return CommandProtocol.from_module(cache, cixstack.cli.status.Status, args)
+        case ["watch", *args]:
+            return CommandProtocol.from_module(cache, cixstack.cli.watch.Watch, args)
+        case ["start", *args]:
+            return CommandProtocol.from_module(cache, cixstack.cli.start.Start, args)
+        case ["match", *args]:
+            return CommandProtocol.from_module(cache, cixstack.cli.match_, args)
+        case ["init", *args]:
+            return CommandProtocol.from_module(cache, cixstack.cli.init, args)
+        case ["config", *args]:
+            return CommandProtocol.from_module(cache, cixstack.cli.config, args)
+        case ["cache", *args]:
+            return CommandProtocol.from_module(cache, cixstack.cli.cache, args)
+        case ["hook", *args]:
+            return CommandProtocol.from_module(cache, cixstack.cli.hook, args)
+        case [alias, *args] if alias.startswith("!"):
+            return SubprocessProtocol(argv=(alias[1:], *args), check=False, shell=False)
+        case []:
+            raise cixstack.errors.MissingCommandError()
+        case [alias, *args]:
+            ctx = cixstack.cli.alias.parse_args(cache, args)
+
+            if alias not in ctx.alias:
+                raise cixstack.errors.InvalidCommandError(alias)
+
+            cmd = ctx.alias[alias]
+            if cmd.startswith("!"):
+                return SubprocessProtocol(argv=(cmd[1:], *args), check=False, shell=True)  # noqa: S604
+
+            return parse_args(cache, [*shlex.split(cmd), *args])
+
+    # If argv is a sequence (as it should always be), this is unreachable
+    raise ValueError(f"Invalid arguments: {argv!r}")
+
+
+def _pretty_format_commands(commands: dict[str, str], *, quote: bool = False) -> str:
+    key_indent_level = min(
+        15,
+        max((len(key) for key in commands), default=10) + 2,
+    )
+    lines = []
+    for key, cmd in commands.items():
+        if quote:
+            cmd = shlex.quote(cmd)
+            is_quoted = {cmd[0], cmd[-1]} <= {"'", '"'}
+        else:
+            is_quoted = False
+
+        lines.append(f"{key + ':':<{key_indent_level + (0 if is_quoted else 1)}}  {cmd}")
+
+    return "\n".join(lines)
+
+
+def main(argv: Sequence[str] = ()) -> int:
+    try:
+        is_debug_trace = bool(json.loads(os.getenv("CIX_DEBUG_TRACE", "false").lower()))
+    except json.JSONDecodeError:
+        sys.stderr.write(
+            f"Invalid CIX_DEBUG_TRACE value will be ignored: {os.getenv('CIX_DEBUG_TRACE')!r}\n"
+        )
+        is_debug_trace = False
+
+    # Always set common termination signals to raise KeyboardInterrupt so finally blocks execute
+    # allowing graceful shutdown with cleanup execution. This is independent of debug_trace.
+    signal.signal(signal.SIGINT, signal.default_int_handler)
+    signal.signal(signal.SIGTERM, signal.default_int_handler)
+    signal.signal(signal.SIGHUP, signal.default_int_handler)
+
+    argv = argv or sys.argv[1:]
+
+    cixstack.utils.init_cache(cixstack.utils.CACHE_DIR, exist_ok=True)
+
+    if ENV_FILE_PATH.is_file():
+        dotenv.load_dotenv(dotenv_path=ENV_FILE_PATH)
+
+    if cwd := cixstack.utils.WORKING_DIRECTORY:
+        os.chdir(cwd)
+
+    with diskcache.Cache(cixstack.utils.CACHE_DIR_DISKCACHE) as cache:
+        if is_debug_trace:
+            protocol = parse_args(cache, argv)
+            try:
+                return protocol.run(cache)
+            except KeyboardInterrupt:
+                # Print traceback for debug_trace mode
+                print("".join(traceback.format_exc()), file=sys.stderr)
+                return 2
+
+        try:
+            protocol = parse_args(cache, argv)
+            return protocol.run(cache)
+        except KeyboardInterrupt:
+            # Graceful shutdown without traceback in normal mode
+            return 2
+        except cixstack.errors.MissingCommandError:
+            print("No command provided. Available commands:", file=sys.stderr)
+            commands = _pretty_format_commands(DEFAULT_COMMANDS, quote=False)
+            print(textwrap.indent(commands, "  "), file=sys.stderr)
+            return 1
+        except cixstack.errors.InvalidCommandError:
+            print("Invalid command provided. Available commands:", file=sys.stderr)
+            commands = _pretty_format_commands(DEFAULT_COMMANDS, quote=False)
+            print(textwrap.indent(commands, "  "), file=sys.stderr)
+            return 1
+        except (cixstack.errors.ConfigError, cixstack.errors.ContextError) as exc:
+            for arg in exc.args:
+                print(arg, file=sys.stderr)
+            print("\nTo create a default config file, run `ci init`.", file=sys.stderr)
+            return 1
+        except cixstack.errors.ConfigFileNotFoundError as exc:
+            print("Config file not found:", *exc.args, file=sys.stderr)
+            return 1
+        finally:
+            # Run is the only command that modifies the cache, so we prune after it runs,
+            # and after it has printed output, so the user gets output as quickly as possible.
+            if len(argv) >= 2 and argv[1] == "run":
+                cixstack.utils.prune_cache_csv(keep_csv_lines=5000)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main(sys.argv[1:]))

cixstack/args.py

@@ -0,0 +1,106 @@
+import argparse
+from enum import Enum
+from pathlib import Path
+
+import cixstack.models.v1
+import cixstack.utils
+
+
+def non_negative_int(value: str) -> int:
+    """Convert string to non-negative integer for argparse validation."""
+    int_value = int(value)
+    if int_value < 0:
+        raise argparse.ArgumentTypeError(f"must be >= 0 (got {int_value})")
+    return int_value
+
+
+class Argument(Enum):
+    STAGES_OR_JOBS = "stages_or_jobs"
+    SERVICE = "service"
+    CONFIG_PATH = "--config-path"
+    REPO_PATH = "--repo-path"
+    QUIET = "--quiet"
+    MAX_JOBS = "--max-jobs"
+    NO_CACHE = "--no-cache"
+    FAIL_FAST = "--fail-fast"
+    RECURSIVE = "--recursive"
+    EXCLUDE = "--exclude"
+
+
+def add_argument(parser: argparse.ArgumentParser, argument: Argument) -> argparse.Action:
+    match argument:
+        case Argument.STAGES_OR_JOBS:
+            return parser.add_argument(
+                "stages_or_jobs",
+                type=str,
+                help="Hook stage",
+                nargs="*",
+                default=(),
+            )
+        case Argument.SERVICE:
+            return parser.add_argument(
+                "service",
+                type=str,
+                help="Service to run",
+            )
+        case Argument.CONFIG_PATH:
+            return parser.add_argument(
+                "--config-path",
+                "-c",
+                type=Path,
+                default=cixstack.utils.DEFAULT_CONFIG_PATH,
+                help="Path to .ci.yaml file",
+            )
+        case Argument.REPO_PATH:
+            return parser.add_argument(
+                "--repo-path",
+                "-r",
+                type=lambda path: Path(path).resolve(),
+                default=cixstack.utils.WORKING_DIRECTORY,
+                help="Path to the repo",
+            )
+        case Argument.QUIET:
+            return parser.add_argument(
+                "--quiet",
+                "-q",
+                action="append_const",
+                help="If set, do not capture output and show full tracebacks",
+            )
+        case Argument.MAX_JOBS:
+            return parser.add_argument(
+                "-n",
+                "--max-jobs",
+                type=non_negative_int,
+                help="Maximum number of jobs to run in parallel. If 0, this is unlimited.",
+                default=0,
+            )
+        case Argument.NO_CACHE:
+            return parser.add_argument(
+                "--no-cache",
+                action="store_true",
+                help="If set, do not use cached results",
+            )
+        case Argument.FAIL_FAST:
+            return parser.add_argument(
+                "--fail-fast",
+                "--ff",
+                action="store_true",
+                help="If set, stop running jobs when one fails",
+            )
+        case Argument.RECURSIVE:
+            return parser.add_argument(
+                "--recursive",
+                action="store_true",
+                help="If set, rerun until all jobs cached",
+            )
+        case Argument.EXCLUDE:
+            return parser.add_argument(
+                "--exclude",
+                "-e",
+                type=cixstack.models.v1.JobState,
+                nargs="*",
+                default=(),
+                help="Exclude jobs with the given state from the output",
+            )
+
+    raise ValueError(f"Unknown argument: {argument}")

cixstack/cli/alias.py

@@ -0,0 +1,40 @@
+import argparse
+from collections.abc import Mapping
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import MutableMapping, Sequence
+    from typing import Any
+
+import attrs
+
+import cixstack.args
+import cixstack.errors
+import cixstack.models.compat
+import cixstack.utils
+
+
+@attrs.frozen
+class Context:
+    alias: Mapping[str, str] = attrs.field(
+        factory=dict,
+        validator=attrs.validators.instance_of(Mapping),
+        converter=dict,
+    )
+
+
+def parse_args(cache: MutableMapping[bytes, Any], argv: Sequence[str]) -> Context:
+    parser = argparse.ArgumentParser()
+
+    cixstack.args.add_argument(parser, cixstack.args.Argument.CONFIG_PATH)
+
+    args, _ = parser.parse_known_args(argv)
+
+    # Validate config path
+    config_path = Path(args.config_path).resolve()
+    if not config_path.is_file():
+        raise cixstack.errors.ConfigFileNotFoundError(config_path)
+
+    config = cixstack.models.compat.read_config(cache, *cixstack.utils.CONFIG_PATHS, config_path)
+    return Context(alias=config.alias)

cixstack/cli/cache.py

@@ -0,0 +1,54 @@
+import argparse
+from typing import TYPE_CHECKING
+
+import argcomplete
+import argcomplete.completers
+import attrs
+
+import cixstack.errors
+import cixstack.utils
+
+if TYPE_CHECKING:
+    from collections.abc import MutableMapping, Sequence
+    from typing import Any
+
+
+VALID_ACTIONS = ("clean", "dir", "prune")
+
+
+@attrs.frozen
+class Context:
+    action: str = attrs.field(validator=attrs.validators.in_(VALID_ACTIONS))
+
+
+def parse_args(cache: MutableMapping[bytes, Any], argv: Sequence[str]) -> Context:
+    del cache  # unused
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "action", type=str, help="Action to perform"
+    ).completer = argcomplete.completers.ChoicesCompleter(VALID_ACTIONS)
+
+    args = parser.parse_args(argv)
+
+    try:
+        ctx = Context(action=args.action)
+    except ValueError as exc:
+        raise cixstack.errors.ContextError(f"Invalid action: {args.action}") from exc
+
+    return ctx
+
+
+def main(cache: MutableMapping[bytes, Any], ctx: Context) -> int:
+    del cache  # unused
+    match ctx.action:
+        case "dir":
+            print(str(cixstack.utils.CACHE_DIR))
+        case "clean":
+            cixstack.utils.prune_cache(keep_csv_lines=0, keep_diskcache=False)
+        case "prune":
+            cixstack.utils.prune_cache(keep_csv_lines=1000, keep_diskcache=True)
+        case _:
+            raise cixstack.errors.ContextError(f"Invalid action: {ctx.action}")
+
+    return 0

cixstack/cli/cleanup.py

@@ -0,0 +1,136 @@
+import argparse
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import argcomplete
+import argcomplete.completers
+import attrs
+
+import cixstack.args
+import cixstack.errors
+import cixstack.models.compat
+import cixstack.models.v2
+import cixstack.utils
+from cixstack.cli.run import Context, Run
+
+if TYPE_CHECKING:
+    from collections.abc import MutableMapping, Sequence
+    from typing import Any
+
+
+def get_valid_jobs_and_stages(config: cixstack.models.v2.RepoConfig) -> tuple[set[str], set[str]]:
+    valid_jobs = {job_name for job_name, job_config in config.jobs.items() if job_config.cleanup}
+    for service_name, service_config in config.services.items():
+        if service_config.cleanup:
+            valid_jobs.add(service_name)
+
+    valid_stages = {
+        stage_name
+        for stage_name, jobs in config.groups.items()
+        if all(job in valid_jobs for job in jobs)
+    }
+    return valid_jobs, valid_stages
+
+
+class Cleanup(Run):
+    @staticmethod
+    def evolve_context(ctx: Context) -> Context:
+        valid_jobs, valid_stages = get_valid_jobs_and_stages(ctx.config)
+        for job in ctx.jobs:
+            if job not in valid_jobs:
+                raise cixstack.errors.ContextError(f"Job has no cleanup command: {job!r}")
+
+        config = attrs.evolve(
+            ctx.config,
+            jobs={
+                **{
+                    job_name: job_config.as_cleanup()
+                    for job_name, job_config in ctx.config.jobs.items()
+                    if job_config.cleanup
+                },
+                **{
+                    service_name: service_config.as_cleanup()
+                    for service_name, service_config in ctx.config.services.items()
+                    if service_config.cleanup
+                },
+            },
+            groups={
+                stage_name: [f"{job}@cleanup" for job in jobs]
+                for stage_name, jobs in ctx.config.groups.items()
+                if stage_name in valid_stages
+            },
+        )
+        config = attrs.evolve(
+            config, jobs={name: job.resolve_env() for name, job in config.jobs.items()}
+        )
+        ctx = attrs.evolve(ctx, config=config)
+        if not ctx.jobs:
+            ctx = attrs.evolve(ctx, stages_or_jobs=ctx.config.jobs.keys())
+
+        file_hashes = cixstack.utils.hash_repo_worktree(ctx.repo_path)
+
+        return attrs.evolve(ctx, config=config, file_hashes=file_hashes)
+
+    @staticmethod
+    def parse_args(cache: MutableMapping[bytes, Any], argv: Sequence[str]) -> Context:
+        parser = argparse.ArgumentParser()
+
+        config = cixstack.models.compat.read_config(cache, *cixstack.utils.CONFIG_PATHS)
+
+        valid_jobs, valid_stages = get_valid_jobs_and_stages(config)
+
+        cixstack.args.add_argument(
+            parser, cixstack.args.Argument.STAGES_OR_JOBS
+        ).completer = argcomplete.completers.ChoicesCompleter(sorted(valid_stages | valid_jobs))
+
+        cixstack.args.add_argument(parser, cixstack.args.Argument.CONFIG_PATH)
+        cixstack.args.add_argument(parser, cixstack.args.Argument.REPO_PATH)
+        cixstack.args.add_argument(parser, cixstack.args.Argument.QUIET)
+        cixstack.args.add_argument(parser, cixstack.args.Argument.MAX_JOBS)
+
+        argcomplete.autocomplete(parser)
+
+        args = parser.parse_args(argv)
+
+        config_path = Path(args.config_path).resolve()
+        repo_path = Path(args.repo_path).resolve()
+
+        config = config.update(cixstack.models.compat.read_config(cache, config_path))
+
+        config = attrs.evolve(
+            config,
+            jobs={
+                **config.jobs,
+                **{
+                    # As this gets re-evolved later in evolve_context, we have to
+                    # add the cleanup in here so it doesn't get lost.
+                    # This is very much a workaround.
+                    name: attrs.evolve(svc.as_cleanup(), cleanup=svc.cleanup)
+                    for name, svc in config.services.items()
+                    if svc.cleanup
+                },
+            },
+            # Clear services to avoid name conflict with jobs
+            # This is fine as we can't run services from here anyways
+            services={},
+        )
+
+        if not repo_path.is_dir():
+            raise cixstack.errors.ContextError(f"Repo path is not a directory: {repo_path}")
+
+        try:
+            ctx = Context(
+                repo_path=repo_path,
+                config=config,
+                stages_or_jobs=args.stages_or_jobs,
+                quiet=len(args.quiet or ()),
+                max_jobs=args.max_jobs,
+                recursive_max_iter=0,
+                no_cache=True,
+                fail_fast=False,
+                file_hashes={},
+            )
+        except ValueError as exc:
+            raise cixstack.errors.ContextError(*exc.args) from exc
+
+        return ctx