pre-commit-ex 4.5.1__py2.py3-none-any.whl

pre_commit/store.py

@@ -0,0 +1,235 @@
+from __future__ import annotations
+
+import contextlib
+import logging
+import os.path
+import sqlite3
+import tempfile
+from collections.abc import Callable
+from collections.abc import Generator
+from collections.abc import Sequence
+
+import pre_commit.constants as C
+from pre_commit import clientlib
+from pre_commit import file_lock
+from pre_commit import git
+from pre_commit.util import CalledProcessError
+from pre_commit.util import clean_path_on_failure
+from pre_commit.util import cmd_output_b
+from pre_commit.util import resource_text
+
+
+logger = logging.getLogger('pre_commit')
+
+
+def _get_default_directory() -> str:
+    """Returns the default directory for the Store.  This is intentionally
+    underscored to indicate that `Store.get_default_directory` is the intended
+    way to get this information.  This is also done so
+    `Store.get_default_directory` can be mocked in tests and
+    `_get_default_directory` can be tested.
+    """
+    ret = os.environ.get('PRE_COMMIT_HOME') or os.path.join(
+        os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~/.cache'),
+        'pre-commit',
+    )
+    return os.path.realpath(ret)
+
+
+_LOCAL_RESOURCES = (
+    'Cargo.toml', 'main.go', 'go.mod', 'main.rs', '.npmignore',
+    'package.json', 'pre-commit-package-dev-1.rockspec',
+    'pre_commit_placeholder_package.gemspec', 'setup.py',
+    'environment.yml', 'Makefile.PL', 'pubspec.yaml',
+    'renv.lock', 'renv/activate.R', 'renv/LICENSE.renv',
+)
+
+
+def _make_local_repo(directory: str) -> None:
+    for resource in _LOCAL_RESOURCES:
+        resource_dirname, resource_basename = os.path.split(resource)
+        contents = resource_text(f'empty_template_{resource_basename}')
+        target_dir = os.path.join(directory, resource_dirname)
+        target_file = os.path.join(target_dir, resource_basename)
+        os.makedirs(target_dir, exist_ok=True)
+        with open(target_file, 'w') as f:
+            f.write(contents)
+
+
+class Store:
+    get_default_directory = staticmethod(_get_default_directory)
+
+    def __init__(self, directory: str | None = None) -> None:
+        self.directory = directory or Store.get_default_directory()
+        self.db_path = os.path.join(self.directory, 'db.db')
+        self.readonly = (
+            os.path.exists(self.directory) and
+            not os.access(self.directory, os.W_OK)
+        )
+
+        if not os.path.exists(self.directory):
+            os.makedirs(self.directory, exist_ok=True)
+            with open(os.path.join(self.directory, 'README'), 'w') as f:
+                f.write(
+                    'This directory is maintained by the pre-commit project.\n'
+                    'Learn more: https://github.com/pre-commit/pre-commit\n',
+                )
+
+        if os.path.exists(self.db_path):
+            return
+        with self.exclusive_lock():
+            # Another process may have already completed this work
+            if os.path.exists(self.db_path):  # pragma: no cover (race)
+                return
+            # To avoid a race where someone ^Cs between db creation and
+            # execution of the CREATE TABLE statement
+            fd, tmpfile = tempfile.mkstemp(dir=self.directory)
+            # We'll be managing this file ourselves
+            os.close(fd)
+            with self.connect(db_path=tmpfile) as db:
+                db.executescript(
+                    'CREATE TABLE repos ('
+                    '    repo TEXT NOT NULL,'
+                    '    ref TEXT NOT NULL,'
+                    '    path TEXT NOT NULL,'
+                    '    PRIMARY KEY (repo, ref)'
+                    ');',
+                )
+                self._create_configs_table(db)
+
+            # Atomic file move
+            os.replace(tmpfile, self.db_path)
+
+    @contextlib.contextmanager
+    def exclusive_lock(self) -> Generator[None]:
+        def blocked_cb() -> None:  # pragma: no cover (tests are in-process)
+            logger.info('Locking pre-commit directory')
+
+        with file_lock.lock(os.path.join(self.directory, '.lock'), blocked_cb):
+            yield
+
+    @contextlib.contextmanager
+    def connect(
+            self,
+            db_path: str | None = None,
+    ) -> Generator[sqlite3.Connection]:
+        db_path = db_path or self.db_path
+        # sqlite doesn't close its fd with its contextmanager >.<
+        # contextlib.closing fixes this.
+        # See: https://stackoverflow.com/a/28032829/812183
+        with contextlib.closing(sqlite3.connect(db_path)) as db:
+            # this creates a transaction
+            with db:
+                yield db
+
+    @classmethod
+    def db_repo_name(cls, repo: str, deps: Sequence[str]) -> str:
+        if deps:
+            return f'{repo}:{",".join(deps)}'
+        else:
+            return repo
+
+    def _new_repo(
+            self,
+            repo: str,
+            ref: str,
+            deps: Sequence[str],
+            make_strategy: Callable[[str], None],
+    ) -> str:
+        original_repo = repo
+        repo = self.db_repo_name(repo, deps)
+
+        def _get_result() -> str | None:
+            # Check if we already exist
+            with self.connect() as db:
+                result = db.execute(
+                    'SELECT path FROM repos WHERE repo = ? AND ref = ?',
+                    (repo, ref),
+                ).fetchone()
+                return result[0] if result else None
+
+        result = _get_result()
+        if result:
+            return result
+        with self.exclusive_lock():
+            # Another process may have already completed this work
+            result = _get_result()
+            if result:  # pragma: no cover (race)
+                return result
+
+            logger.info(f'Initializing environment for {repo}.')
+
+            directory = tempfile.mkdtemp(prefix='repo', dir=self.directory)
+            with clean_path_on_failure(directory):
+                make_strategy(directory)
+
+            # Update our db with the created repo
+            with self.connect() as db:
+                db.execute(
+                    'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)',
+                    [repo, ref, directory],
+                )
+
+            clientlib.warn_for_stages_on_repo_init(original_repo, directory)
+
+        return directory
+
+    def _complete_clone(self, ref: str, git_cmd: Callable[..., None]) -> None:
+        """Perform a complete clone of a repository and its submodules """
+
+        git_cmd('fetch', 'origin', '--tags')
+        git_cmd('checkout', ref)
+        git_cmd('submodule', 'update', '--init', '--recursive')
+
+    def _shallow_clone(self, ref: str, git_cmd: Callable[..., None]) -> None:
+        """Perform a shallow clone of a repository and its submodules """
+
+        git_config = 'protocol.version=2'
+        git_cmd('-c', git_config, 'fetch', 'origin', ref, '--depth=1')
+        git_cmd('checkout', 'FETCH_HEAD')
+        git_cmd(
+            '-c', git_config, 'submodule', 'update', '--init', '--recursive',
+            '--depth=1',
+        )
+
+    def clone(self, repo: str, ref: str, deps: Sequence[str] = ()) -> str:
+        """Clone the given url and checkout the specific ref."""
+
+        def clone_strategy(directory: str) -> None:
+            git.init_repo(directory, repo)
+            env = git.no_git_env()
+
+            def _git_cmd(*args: str) -> None:
+                cmd_output_b('git', *args, cwd=directory, env=env)
+
+            try:
+                self._shallow_clone(ref, _git_cmd)
+            except CalledProcessError:
+                self._complete_clone(ref, _git_cmd)
+
+        return self._new_repo(repo, ref, deps, clone_strategy)
+
+    def make_local(self, deps: Sequence[str]) -> str:
+        return self._new_repo(
+            'local', C.LOCAL_REPO_VERSION, deps, _make_local_repo,
+        )
+
+    def _create_configs_table(self, db: sqlite3.Connection) -> None:
+        db.executescript(
+            'CREATE TABLE IF NOT EXISTS configs ('
+            '   path TEXT NOT NULL,'
+            '   PRIMARY KEY (path)'
+            ');',
+        )
+
+    def mark_config_used(self, path: str) -> None:
+        if self.readonly:  # pragma: win32 no cover
+            return
+        path = os.path.realpath(path)
+        # don't insert config files that do not exist
+        if not os.path.exists(path):
+            return
+        with self.connect() as db:
+            # TODO: eventually remove this and only create in _create
+            self._create_configs_table(db)
+            db.execute('INSERT OR IGNORE INTO configs VALUES (?)', (path,))

pre_commit/util.py

@@ -0,0 +1,239 @@
+from __future__ import annotations
+
+import contextlib
+import errno
+import importlib.resources
+import os.path
+import shutil
+import stat
+import subprocess
+import sys
+from collections.abc import Callable
+from collections.abc import Generator
+from types import TracebackType
+from typing import Any
+
+from pre_commit import parse_shebang
+
+
+def force_bytes(exc: Any) -> bytes:
+    with contextlib.suppress(TypeError):
+        return bytes(exc)
+    with contextlib.suppress(Exception):
+        return str(exc).encode()
+    return f'<unprintable {type(exc).__name__} object>'.encode()
+
+
+@contextlib.contextmanager
+def clean_path_on_failure(path: str) -> Generator[None]:
+    """Cleans up the directory on an exceptional failure."""
+    try:
+        yield
+    except BaseException:
+        if os.path.exists(path):
+            rmtree(path)
+        raise
+
+
+def resource_text(filename: str) -> str:
+    files = importlib.resources.files('pre_commit.resources')
+    return files.joinpath(filename).read_text()
+
+
+def make_executable(filename: str) -> None:
+    original_mode = os.stat(filename).st_mode
+    new_mode = original_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+    os.chmod(filename, new_mode)
+
+
+class CalledProcessError(RuntimeError):
+    def __init__(
+            self,
+            returncode: int,
+            cmd: tuple[str, ...],
+            stdout: bytes,
+            stderr: bytes | None,
+    ) -> None:
+        super().__init__(returncode, cmd, stdout, stderr)
+        self.returncode = returncode
+        self.cmd = cmd
+        self.stdout = stdout
+        self.stderr = stderr
+
+    def __bytes__(self) -> bytes:
+        def _indent_or_none(part: bytes | None) -> bytes:
+            if part:
+                return b'\n    ' + part.replace(b'\n', b'\n    ').rstrip()
+            else:
+                return b' (none)'
+
+        return b''.join((
+            f'command: {self.cmd!r}\n'.encode(),
+            f'return code: {self.returncode}\n'.encode(),
+            b'stdout:', _indent_or_none(self.stdout), b'\n',
+            b'stderr:', _indent_or_none(self.stderr),
+        ))
+
+    def __str__(self) -> str:
+        return self.__bytes__().decode()
+
+
+def _setdefault_kwargs(kwargs: dict[str, Any]) -> None:
+    for arg in ('stdin', 'stdout', 'stderr'):
+        kwargs.setdefault(arg, subprocess.PIPE)
+
+
+def _oserror_to_output(e: OSError) -> tuple[int, bytes, None]:
+    return 1, force_bytes(e).rstrip(b'\n') + b'\n', None
+
+
+def cmd_output_b(
+        *cmd: str,
+        check: bool = True,
+        **kwargs: Any,
+) -> tuple[int, bytes, bytes | None]:
+    _setdefault_kwargs(kwargs)
+
+    try:
+        cmd = parse_shebang.normalize_cmd(cmd, env=kwargs.get('env'))
+    except parse_shebang.ExecutableNotFoundError as e:
+        returncode, stdout_b, stderr_b = e.to_output()
+    else:
+        try:
+            proc = subprocess.Popen(cmd, **kwargs)
+        except OSError as e:
+            returncode, stdout_b, stderr_b = _oserror_to_output(e)
+        else:
+            stdout_b, stderr_b = proc.communicate()
+            returncode = proc.returncode
+
+    if check and returncode:
+        raise CalledProcessError(returncode, cmd, stdout_b, stderr_b)
+
+    return returncode, stdout_b, stderr_b
+
+
+def cmd_output(*cmd: str, **kwargs: Any) -> tuple[int, str, str | None]:
+    returncode, stdout_b, stderr_b = cmd_output_b(*cmd, **kwargs)
+    stdout = stdout_b.decode() if stdout_b is not None else None
+    stderr = stderr_b.decode() if stderr_b is not None else None
+    return returncode, stdout, stderr
+
+
+if sys.platform != 'win32':  # pragma: win32 no cover
+    from os import openpty
+    import termios
+
+    class Pty:
+        def __init__(self) -> None:
+            self.r: int | None = None
+            self.w: int | None = None
+
+        def __enter__(self) -> Pty:
+            self.r, self.w = openpty()
+
+            # tty flags normally change \n to \r\n
+            attrs = termios.tcgetattr(self.w)
+            assert isinstance(attrs[1], int)
+            attrs[1] &= ~(termios.ONLCR | termios.OPOST)
+            termios.tcsetattr(self.w, termios.TCSANOW, attrs)
+
+            return self
+
+        def close_w(self) -> None:
+            if self.w is not None:
+                os.close(self.w)
+                self.w = None
+
+        def close_r(self) -> None:
+            assert self.r is not None
+            os.close(self.r)
+            self.r = None
+
+        def __exit__(
+                self,
+                exc_type: type[BaseException] | None,
+                exc_value: BaseException | None,
+                traceback: TracebackType | None,
+        ) -> None:
+            self.close_w()
+            self.close_r()
+
+    def cmd_output_p(
+            *cmd: str,
+            check: bool = True,
+            **kwargs: Any,
+    ) -> tuple[int, bytes, bytes | None]:
+        assert check is False
+        assert kwargs['stderr'] == subprocess.STDOUT, kwargs['stderr']
+        _setdefault_kwargs(kwargs)
+
+        try:
+            cmd = parse_shebang.normalize_cmd(cmd)
+        except parse_shebang.ExecutableNotFoundError as e:
+            return e.to_output()
+
+        with open(os.devnull) as devnull, Pty() as pty:
+            assert pty.r is not None
+            kwargs.update({'stdin': devnull, 'stdout': pty.w, 'stderr': pty.w})
+            try:
+                proc = subprocess.Popen(cmd, **kwargs)
+            except OSError as e:
+                return _oserror_to_output(e)
+
+            pty.close_w()
+
+            buf = b''
+            while True:
+                try:
+                    bts = os.read(pty.r, 4096)
+                except OSError as e:
+                    if e.errno == errno.EIO:
+                        bts = b''
+                    else:
+                        raise
+                else:
+                    buf += bts
+                if not bts:
+                    break
+
+        return proc.wait(), buf, None
+else:  # pragma: no cover
+    cmd_output_p = cmd_output_b
+
+
+def _handle_readonly(
+        func: Callable[[str], object],
+        path: str,
+        exc: BaseException,
+) -> None:
+    if (
+            func in (os.rmdir, os.remove, os.unlink) and
+            isinstance(exc, OSError) and
+            exc.errno in {errno.EACCES, errno.EPERM}
+    ):
+        for p in (path, os.path.dirname(path)):
+            os.chmod(p, os.stat(p).st_mode | stat.S_IWUSR)
+        func(path)
+    else:
+        raise
+
+
+if sys.version_info < (3, 12):  # pragma: <3.12 cover
+    def _handle_readonly_old(
+        func: Callable[[str], object],
+        path: str,
+        excinfo: tuple[type[BaseException], BaseException, TracebackType],
+    ) -> None:
+        return _handle_readonly(func, path, excinfo[1])
+
+    def rmtree(path: str) -> None:
+        shutil.rmtree(path, ignore_errors=False, onerror=_handle_readonly_old)
+else:  # pragma: >=3.12 cover
+    def rmtree(path: str) -> None:
+        """On windows, rmtree fails for readonly dirs."""
+        shutil.rmtree(path, ignore_errors=False, onexc=_handle_readonly)
+
+
+def win_exe(s: str) -> str:
+    return s if sys.platform != 'win32' else f'{s}.exe'

pre_commit/xargs.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import concurrent.futures
+import contextlib
+import math
+import multiprocessing
+import os
+import subprocess
+import sys
+from collections.abc import Callable
+from collections.abc import Generator
+from collections.abc import Iterable
+from collections.abc import MutableMapping
+from collections.abc import Sequence
+from typing import Any
+from typing import TypeVar
+
+from pre_commit import parse_shebang
+from pre_commit.util import cmd_output_b
+from pre_commit.util import cmd_output_p
+
+TArg = TypeVar('TArg')
+TRet = TypeVar('TRet')
+
+
+def cpu_count() -> int:
+    try:
+        # On systems that support it, this will return a more accurate count of
+        # usable CPUs for the current process, which will take into account
+        # cgroup limits
+        return len(os.sched_getaffinity(0))
+    except AttributeError:
+        pass
+
+    try:
+        return multiprocessing.cpu_count()
+    except NotImplementedError:
+        return 1
+
+
+def _environ_size(_env: MutableMapping[str, str] | None = None) -> int:
+    environ = _env if _env is not None else getattr(os, 'environb', os.environ)
+    size = 8 * len(environ)  # number of pointers in `envp`
+    for k, v in environ.items():
+        size += len(k) + len(v) + 2  # c strings in `envp`
+    return size
+
+
+def _get_platform_max_length() -> int:  # pragma: no cover (platform specific)
+    if os.name == 'posix':
+        maximum = os.sysconf('SC_ARG_MAX') - 2048 - _environ_size()
+        maximum = max(min(maximum, 2 ** 17), 2 ** 12)
+        return maximum
+    elif os.name == 'nt':
+        return 2 ** 15 - 2048  # UNICODE_STRING max - headroom
+    else:
+        # posix minimum
+        return 2 ** 12
+
+
+def _command_length(*cmd: str) -> int:
+    full_cmd = ' '.join(cmd)
+
+    # win32 uses the amount of characters, more details at:
+    # https://github.com/pre-commit/pre-commit/pull/839
+    if sys.platform == 'win32':
+        return len(full_cmd.encode('utf-16le')) // 2
+    else:
+        return len(full_cmd.encode(sys.getfilesystemencoding()))
+
+
+class ArgumentTooLongError(RuntimeError):
+    pass
+
+
+def partition(
+        cmd: Sequence[str],
+        varargs: Sequence[str],
+        target_concurrency: int,
+        _max_length: int | None = None,
+) -> tuple[tuple[str, ...], ...]:
+    _max_length = _max_length or _get_platform_max_length()
+
+    # Generally, we try to partition evenly into at least `target_concurrency`
+    # partitions, but we don't want a bunch of tiny partitions.
+    max_args = max(4, math.ceil(len(varargs) / target_concurrency))
+
+    cmd = tuple(cmd)
+    ret = []
+
+    ret_cmd: list[str] = []
+    # Reversed so arguments are in order
+    varargs = list(reversed(varargs))
+
+    total_length = _command_length(*cmd) + 1
+    while varargs:
+        arg = varargs.pop()
+
+        arg_length = _command_length(arg) + 1
+        if (
+                total_length + arg_length <= _max_length and
+                len(ret_cmd) < max_args
+        ):
+            ret_cmd.append(arg)
+            total_length += arg_length
+        elif not ret_cmd:
+            raise ArgumentTooLongError(arg)
+        else:
+            # We've exceeded the length, yield a command
+            ret.append(cmd + tuple(ret_cmd))
+            ret_cmd = []
+            total_length = _command_length(*cmd) + 1
+            varargs.append(arg)
+
+    ret.append(cmd + tuple(ret_cmd))
+
+    return tuple(ret)
+
+
+@contextlib.contextmanager
+def _thread_mapper(maxsize: int) -> Generator[
+    Callable[[Callable[[TArg], TRet], Iterable[TArg]], Iterable[TRet]],
+]:
+    if maxsize == 1:
+        yield map
+    else:
+        with concurrent.futures.ThreadPoolExecutor(maxsize) as ex:
+            yield ex.map
+
+
+def xargs(
+        cmd: tuple[str, ...],
+        varargs: Sequence[str],
+        *,
+        color: bool = False,
+        target_concurrency: int = 1,
+        _max_length: int = _get_platform_max_length(),
+        **kwargs: Any,
+) -> tuple[int, bytes]:
+    """A simplified implementation of xargs.
+
+    color: Make a pty if on a platform that supports it
+    target_concurrency: Target number of partitions to run concurrently
+    """
+    cmd_fn = cmd_output_p if color else cmd_output_b
+    retcode = 0
+    stdout = b''
+
+    try:
+        cmd = parse_shebang.normalize_cmd(cmd)
+    except parse_shebang.ExecutableNotFoundError as e:
+        return e.to_output()[:2]
+
+    # on windows, batch files have a separate length limit than windows itself
+    if (
+            sys.platform == 'win32' and
+            cmd[0].lower().endswith(('.bat', '.cmd'))
+    ):  # pragma: win32 cover
+        # this is implementation details but the command gets translated into
+        # full/path/to/cmd.exe /c *cmd
+        cmd_exe = parse_shebang.find_executable('cmd.exe')
+        # 1024 is additionally subtracted to give headroom for further
+        # expansion inside the batch file
+        _max_length = 8192 - len(cmd_exe) - len(' /c ') - 1024
+
+    partitions = partition(cmd, varargs, target_concurrency, _max_length)
+
+    def run_cmd_partition(
+            run_cmd: tuple[str, ...],
+    ) -> tuple[int, bytes, bytes | None]:
+        return cmd_fn(
+            *run_cmd, check=False, stderr=subprocess.STDOUT, **kwargs,
+        )
+
+    threads = min(len(partitions), target_concurrency)
+    with _thread_mapper(threads) as thread_map:
+        results = thread_map(run_cmd_partition, partitions)
+
+        for proc_retcode, proc_out, _ in results:
+            if abs(proc_retcode) > abs(retcode):
+                retcode = proc_retcode
+            stdout += proc_out
+
+    return retcode, stdout

pre_commit/yaml.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import functools
+from typing import Any
+
+import yaml
+
+Loader = getattr(yaml, 'CSafeLoader', yaml.SafeLoader)
+yaml_compose = functools.partial(yaml.compose, Loader=Loader)
+yaml_load = functools.partial(yaml.load, Loader=Loader)
+Dumper = getattr(yaml, 'CSafeDumper', yaml.SafeDumper)
+
+
+def yaml_dump(o: Any, **kwargs: Any) -> str:
+    # when python/mypy#1484 is solved, this can be `functools.partial`
+    return yaml.dump(
+        o, Dumper=Dumper, default_flow_style=False, indent=4, sort_keys=False,
+        **kwargs,
+    )