omerta 0.1.0__py3-none-any.whl

omerta/__init__.py

@@ -0,0 +1,5 @@
+"""Fail when a value of your env files is in git."""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"

omerta/check.py

@@ -0,0 +1,125 @@
+"""``omerta check``: the index content of the staged changes, of given paths, or of every entry."""
+
+from __future__ import annotations
+
+import os
+
+from omerta.errors import OmertaError
+from omerta.git import GITLINK, Entry, Git
+from omerta.output import Finding
+from omerta.patterns import Pattern, is_env_file
+from omerta.search import Matcher, Target, line_of
+
+
+def select(git: Git, index: list[Entry], paths: list[str], everything: bool) -> list[Entry]:
+    """The index entries that ``check`` searches.
+
+    A PATH selects the entry at that path, or with a folder the entries under
+    it, matched exactly, as literal pathspecs are. The paths are matched here,
+    not given to git, so that none goes on git's command line.
+
+    Args:
+        git: git, in the repository.
+        index: Every index entry.
+        paths: The PATH arguments, relative to the folder omerta runs in.
+        everything: ``--all``: every entry.
+
+    Raises:
+        OmertaError: A path is outside the repository.
+    """
+    if everything:
+        return index
+    if paths:
+        selected = [_from_root(git, path) for path in paths]
+        return [
+            entry
+            for entry in index
+            if any(
+                not chosen or entry.path == chosen or entry.path.startswith(chosen + b"/")
+                for chosen in selected
+            )
+        ]
+    staged = git.staged_paths()
+    return [entry for entry in index if entry.path in staged]
+
+
+def _from_root(git: Git, path: str) -> bytes:
+    """A PATH argument as an index path: from the root, with ``/``; empty for the root."""
+    full = os.path.abspath(os.path.join(git.folder, path))
+    try:
+        relative = os.path.relpath(full, os.path.abspath(git.top))
+    except ValueError:  # on another drive
+        relative = os.pardir
+    if relative == os.pardir or relative.startswith(os.pardir + os.sep):
+        raise OmertaError(f"{path} is outside the repository")
+    return b"" if relative == os.curdir else os.fsencode(relative.replace(os.sep, "/"))
+
+
+def run(
+    git: Git,
+    entries: list[Entry],
+    targets: list[Target],
+    patterns: tuple[Pattern, ...],
+    findings: list[Finding],
+) -> None:
+    """Search the entries, adding what is found to ``findings``.
+
+    An entry that matches the patterns is a tracked env file, and a path that
+    holds a value is a finding of its own. A gitlink (a submodule) has no
+    content to search. An unmerged entry is searched in each of its stages,
+    and a value is reported at its first occurrence in the lowest stage that
+    holds it.
+
+    Raises:
+        OmertaError: An entry's object cannot be read.
+    """
+    matcher = Matcher(targets)
+    for path in sorted({entry.path for entry in entries}):
+        for index, hit in matcher.search(path).items():
+            findings.append(_found(targets[index], hit.form, path=path, in_path=True))
+    entries = sorted(
+        (entry for entry in entries if entry.mode != GITLINK),
+        key=lambda entry: (entry.path, entry.stage),
+    )
+    tracked: set[bytes] = set()
+    for entry in entries:
+        if entry.path not in tracked and is_env_file(entry.path, patterns):
+            tracked.add(entry.path)
+            findings.append(Finding(path=entry.path))
+    if not targets:
+        return
+    by_oid: dict[str, list[Entry]] = {}
+    for entry in entries:
+        by_oid.setdefault(entry.oid, []).append(entry)
+    found: dict[tuple[bytes, int], tuple[int, Finding]] = {}
+    try:
+        for oid, _, data in git.objects(list(by_oid)):
+            hits = matcher.search(data)
+            for entry in by_oid[oid]:
+                for index, hit in hits.items():
+                    key = (entry.path, index)
+                    if key in found and found[key][0] <= entry.stage:
+                        continue
+                    line = line_of(data, hit.position)
+                    found[key] = (
+                        entry.stage,
+                        _found(targets[index], hit.form, path=entry.path, line=line),
+                    )
+    finally:
+        # What was found before an error is reported with it.
+        findings.extend(finding for _, finding in found.values())
+
+
+def _found(
+    target: Target, form: str, *, path: bytes, line: int | None = None, in_path: bool = False
+) -> Finding:
+    """A finding of a target in an entry: in its content at a line, or in its path."""
+    return Finding(
+        path=path,
+        line=line,
+        in_path=in_path,
+        variable=target.variable,
+        env_file=target.env_file,
+        env_order=target.env_order,
+        form=form,
+    )

omerta/cli.py

@@ -0,0 +1,214 @@
+"""The command line: ``omerta check`` and ``omerta history``."""
+
+from __future__ import annotations
+
+import argparse
+import errno
+import os
+import sys
+import traceback
+from pathlib import Path
+from typing import TextIO
+
+from omerta import __version__, check, config, envfiles, history
+from omerta.errors import OmertaError
+from omerta.git import WINDOWS, Git, GitError, environment, find_executable, find_repository
+from omerta.output import Finding, json_line, sort_key, text_line
+from omerta.search import Target, redact, targets
+
+_ISSUES = "https://github.com/shcherbak-ai/omerta/issues"
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Run omerta with the arguments of the command line, or ``argv``.
+
+    Returns:
+        The exit code: 0 when nothing is found, 1 when a value or a tracked env
+        file is found, and 2 on an error, which wins over findings.
+    """
+    parser, check_parser = _parser()
+    try:
+        args = parser.parse_args(argv)
+        if args.command == "check" and args.all and args.paths:
+            check_parser.error("PATH arguments cannot be given with --all")
+    except SystemExit as exit_:
+        return exit_.code if isinstance(exit_.code, int) else 0 if exit_.code is None else 2
+    findings: list[Finding] = []
+    searched: list[Target] = []
+
+    def hide(data: bytes) -> bytes:
+        """``data`` with each value of the env files replaced by its variable's name."""
+        return redact(data, searched)
+
+    try:
+        _run(args, findings, searched)
+        error = None
+    except GitError as failure:
+        error = failure.describe(hide)
+    except OmertaError as failure:
+        error = str(failure)
+    except Exception as failure:  # noqa: BLE001 - reported without its text, which may hold a value
+        frame = traceback.extract_tb(failure.__traceback__)[-1]
+        error = (
+            f"unexpected {type(failure).__name__} at {Path(frame.filename).name}:{frame.lineno}; "
+            f"please report it at {_ISSUES}"
+        )
+    if not args.quiet:
+        line = json_line if args.json else text_line
+        ordered = sorted(findings, key=sort_key)
+        _write(sys.stdout, "".join(line(each, hide) + "\n" for each in ordered))
+    if error is not None:
+        # A path or an env file's name in the message may hold a value too.
+        message = hide(f"omerta: {error}\n".encode("utf-8", "surrogateescape"))
+        _write(sys.stderr, message.decode("utf-8", "backslashreplace"))
+        return 2
+    return 1 if findings else 0
+
+
+def _run(args: argparse.Namespace, findings: list[Finding], searched: list[Target]) -> None:
+    """Run a command, adding what it finds to ``findings``, and its targets to ``searched``."""
+    start = os.getcwd()
+    folder = start
+    for given in args.folders:
+        folder = os.path.join(folder, given)
+        if not os.path.isdir(folder):
+            raise OmertaError(f"cannot change to {given}: no such folder")
+    # Where its links lead, as git sees it: git's `--show-cdup` climbs from there.
+    folder = os.path.realpath(folder)
+    executable = find_executable(
+        os.environ.get("PATH", ""), [start, folder], find_repository(folder)
+    )
+    git = Git(executable, folder, environment())
+    git.open()
+    if args.command == "history" and git.is_shallow():
+        raise OmertaError(
+            "this repository is a shallow clone, whose history stops at a cut: fetch all of it "
+            "(in CI, check out with fetch-depth: 0)"
+        )
+    top = Path(git.top)
+    settings = config.load(
+        top,
+        env_files=args.env_files,
+        min_chars=args.min_chars,
+        ignore=args.ignore,
+        encoded=args.encoded,
+    )
+    index = git.index()
+    sources = envfiles.find(top, settings.patterns, {entry.path for entry in index})
+    searched.extend(targets(sources, settings))
+    if args.command == "check":
+        entries = check.select(git, index, args.paths, args.all)
+        check.run(git, entries, searched, settings.patterns, findings)
+    else:
+        history.run(git, searched, settings.patterns, findings)
+
+
+def _parser() -> tuple[argparse.ArgumentParser, argparse.ArgumentParser]:
+    """The parser of the command line, and that of ``check``."""
+    parser = argparse.ArgumentParser(
+        prog="omerta",
+        description="Fail when a value of your env files, e.g. an API key in .env, is in git.",
+    )
+    parser.add_argument(
+        "-C",
+        dest="folders",
+        action="append",
+        default=[],
+        metavar="PATH",
+        help="run as if started in PATH, as git -C does",
+    )
+    parser.add_argument("--version", action="version", version=f"omerta {__version__}")
+    commands = parser.add_subparsers(dest="command", metavar="COMMAND", required=True)
+    options = argparse.ArgumentParser(add_help=False)
+    options.add_argument(
+        "--min-chars",
+        type=_positive,
+        metavar="N",
+        help="the shortest value searched for, in characters (default: 8)",
+    )
+    options.add_argument(
+        "--env-file",
+        dest="env_files",
+        action="append",
+        metavar="PATTERN",
+        help="an env file pattern, repeatable (default: .env and .env.*)",
+    )
+    options.add_argument(
+        "--ignore",
+        action="append",
+        default=[],
+        metavar="NAME",
+        help="a variable not searched for, repeatable",
+    )
+    options.add_argument(
+        "--encoded",
+        action=argparse.BooleanOptionalAction,
+        help="search the URL, JSON and base64 forms too (default), or the plain forms only",
+    )
+    options.add_argument("--json", action="store_true", help="findings as JSON Lines")
+    options.add_argument(
+        "-q", "--quiet", action="store_true", help="print no findings: the exit code says it"
+    )
+    found = commands.add_parser(
+        "check",
+        parents=[options],
+        help="search the staged changes, PATHs, or every index entry",
+        description=(
+            "Search the index content of the staged changes, of the PATHs given, or with --all "
+            "of every entry."
+        ),
+    )
+    found.add_argument("paths", nargs="*", metavar="PATH", help="paths of index entries")
+    found.add_argument("--all", action="store_true", help="search every index entry")
+    commands.add_parser(
+        "history",
+        parents=[options],
+        help="search every blob and message reachable from a ref",
+        description=(
+            "Search every blob reachable from a ref, and the messages of the commits and tags."
+        ),
+    )
+    return parser, found
+
+
+def _positive(text: str) -> int:
+    """An integer of at least 1, from the command line."""
+    try:
+        number = int(text)
+    except ValueError:
+        number = 0
+    if number < 1:
+        raise argparse.ArgumentTypeError(f"{text!r} is not an integer of at least 1")
+    return number
+
+
+def _write(stream: TextIO, text: str) -> None:
+    """Write text to a stream in UTF-8, with ``\\n`` line ends, whatever its encoding.
+
+    A stream whose reader has gone (``omerta history | head``) takes nothing
+    more: the rest is dropped, and so is what Python would flush into it at exit.
+    """
+    buffer = getattr(stream, "buffer", None)
+    try:
+        if buffer is None:
+            stream.write(text)
+            return
+        stream.flush()
+        buffer.write(text.encode("utf-8"))
+        buffer.flush()
+    except OSError as error:
+        if not _reader_gone(error):
+            raise
+        devnull = os.open(os.devnull, os.O_WRONLY)
+        os.dup2(devnull, stream.fileno())
+        os.close(devnull)
+
+
+def _reader_gone(error: OSError) -> bool:
+    """Whether a write failed because the reader of its pipe has gone.
+
+    Other platforms raise :class:`BrokenPipeError`. Windows raises it too
+    for a reader that goes while omerta writes, but ``EINVAL`` for one that
+    went before the write.
+    """
+    return isinstance(error, BrokenPipeError) or (WINDOWS and error.errno == errno.EINVAL)

omerta/config.py

@@ -0,0 +1,120 @@
+"""The settings of a run: the defaults, the configuration file, and the options."""
+
+from __future__ import annotations
+
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from omerta.errors import OmertaError
+from omerta.patterns import DEFAULT_PATTERNS, Pattern
+
+DEFAULT_MIN_CHARS = 8
+
+_KEYS = ("env-files", "min-chars", "ignore", "encoded")
+
+
+@dataclass(frozen=True)
+class Settings:
+    """What a run searches for: the env files, and which of their values."""
+
+    patterns: tuple[Pattern, ...]
+    min_chars: int
+    ignore: frozenset[str]
+    encoded: bool
+
+
+def load(
+    top: Path,
+    *,
+    env_files: list[str] | None = None,
+    min_chars: int | None = None,
+    ignore: list[str] | None = None,
+    encoded: bool | None = None,
+) -> Settings:
+    """The settings, from the options given, the configuration, and the defaults.
+
+    An option overrides the setting of the same name in the configuration,
+    which overrides the default, except ``ignore``, whose names are added to
+    the configured ones.
+
+    Args:
+        top: The root of the repository, where the configuration file is.
+        env_files: The env file patterns of the command line, if any.
+        min_chars: The ``--min-chars`` of the command line, if given.
+        ignore: The variable names of the command line.
+        encoded: ``--encoded`` or ``--no-encoded``, if given.
+
+    Raises:
+        OmertaError: The configuration cannot be read or has a wrong setting.
+    """
+    configured = _read(top)
+    patterns = env_files or configured.get("env-files") or list(DEFAULT_PATTERNS)
+    return Settings(
+        patterns=tuple(Pattern(text) for text in patterns),
+        min_chars=min_chars or configured.get("min-chars", DEFAULT_MIN_CHARS),
+        ignore=frozenset([*configured.get("ignore", []), *(ignore or [])]),
+        encoded=configured.get("encoded", True) if encoded is None else encoded,
+    )
+
+
+def _read(top: Path) -> dict[str, Any]:
+    """The settings of ``[tool.omerta]`` in ``pyproject.toml`` or of ``.omerta.toml``."""
+    pyproject = _toml(top / "pyproject.toml")
+    tool = pyproject.get("tool") if pyproject is not None else None
+    in_pyproject = tool.get("omerta") if isinstance(tool, dict) else None
+    standalone = _toml(top / ".omerta.toml")
+    if in_pyproject is not None and standalone is not None:
+        raise OmertaError(
+            "settings are in both pyproject.toml ([tool.omerta]) and .omerta.toml: keep one"
+        )
+    if in_pyproject is not None:
+        if not isinstance(in_pyproject, dict):
+            raise OmertaError("pyproject.toml: tool.omerta must be a table")
+        return _validate(in_pyproject, "pyproject.toml [tool.omerta]")
+    if standalone is not None:
+        return _validate(standalone, ".omerta.toml")
+    return {}
+
+
+def _toml(path: Path) -> dict[str, Any] | None:
+    """The document of a TOML file, or ``None`` where there is no such file."""
+    if not path.is_file():
+        return None
+    try:
+        text = path.read_bytes().decode("utf-8")
+    except UnicodeDecodeError:
+        raise OmertaError(f"{path.name} is not UTF-8") from None
+    except OSError as error:
+        raise OmertaError(f"{path.name} cannot be read ({error.strerror})") from None
+    try:
+        return tomllib.loads(text)
+    except tomllib.TOMLDecodeError as error:
+        raise OmertaError(f"{path.name}: {error}") from None
+
+
+def _validate(table: dict[str, Any], source: str) -> dict[str, Any]:
+    """The settings of a configuration table, checked key by key."""
+    for key, value in table.items():
+        if key not in _KEYS:
+            raise OmertaError(f"{source}: unknown setting {key!r}")
+        if key == "env-files" and not (_strings(value) and value):
+            raise OmertaError(f"{source}: env-files must be a non-empty list of patterns")
+        if key == "min-chars" and (type(value) is not int or value < 1):
+            raise OmertaError(f"{source}: min-chars must be an integer of at least 1")
+        if key == "ignore" and not _strings(value):
+            raise OmertaError(f"{source}: ignore must be a list of variable names")
+        if key == "encoded" and type(value) is not bool:
+            raise OmertaError(f"{source}: encoded must be true or false")
+    for pattern in table.get("env-files", []):
+        try:
+            Pattern(pattern)
+        except OmertaError as error:
+            raise OmertaError(f"{source}: {error}") from None
+    return table
+
+
+def _strings(value: object) -> bool:
+    """Whether a value is a list of strings."""
+    return isinstance(value, list) and all(isinstance(item, str) for item in value)

omerta/envfiles.py

@@ -0,0 +1,153 @@
+"""The env files of the working tree, and the variables they set."""
+
+from __future__ import annotations
+
+import codecs
+import fnmatch
+import io
+import os
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+from dotenv.parser import parse_stream
+
+from omerta.errors import OmertaError
+from omerta.output import quote
+from omerta.patterns import Pattern, is_template
+
+
+@dataclass(frozen=True)
+class EnvFile:
+    """An env file that git does not track, a source of values."""
+
+    path: str
+    """Its path relative to the root of the repository, with ``/``."""
+
+    absolute: Path
+
+
+def find(top: Path, patterns: tuple[Pattern, ...], tracked: set[bytes]) -> list[EnvFile]:
+    """The env files of the working tree that git does not track.
+
+    Each pattern's folders are walked with :func:`os.scandir`: ``.git`` is never
+    entered, nor a folder that holds a ``.git`` of its own (a submodule, a
+    nested repository), nor a symbolic link to a folder; a symbolic link to a
+    file is read through. A folder that matches a pattern is no env file, and
+    neither is a template.
+
+    Args:
+        top: The root of the repository.
+        patterns: The env file patterns, in order.
+        tracked: The paths of the index entries, as git gives them.
+
+    Returns:
+        The env files in the order of the patterns, then of their paths.
+
+    Raises:
+        OmertaError: A folder on the way cannot be listed.
+    """
+    found: dict[str, EnvFile] = {}
+    for pattern in patterns:
+        for parts in sorted(set(_walk(top, (), pattern.parts))):
+            path = "/".join(parts)
+            if path in found or is_template(parts[-1]) or os.fsencode(path) in tracked:
+                continue
+            found[path] = EnvFile(path, top.joinpath(*parts))
+    return list(found.values())
+
+
+def read(env_file: EnvFile) -> list[tuple[str, str | None]]:
+    """The variables an env file sets, with their values, in the order of the file.
+
+    The file's bytes are decoded as UTF-16 after a UTF-16 byte order mark, and
+    as UTF-8 otherwise, with or without one; line ends are then made ``\\n``, as
+    Python's text mode reads a file. python-dotenv parses the text: quotes,
+    ``export`` and comments, with no ``${VAR}`` expanded. A variable given twice
+    is listed twice. A key without ``=`` has the value ``None``.
+
+    Raises:
+        OmertaError: The file cannot be read, is not UTF-8 or UTF-16, or has a
+            statement that python-dotenv cannot parse. The error names the file
+            and the line, never its text.
+    """
+    name = quote(os.fsencode(env_file.path))
+    try:
+        data = env_file.absolute.read_bytes()
+    except OSError as error:
+        raise OmertaError(f"{name} cannot be read ({error.strerror})") from None
+    text = _decode(data)
+    if text is None:
+        raise OmertaError(f"{name} is neither UTF-8 nor UTF-16 text")
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+    bindings = []
+    for binding in parse_stream(io.StringIO(text)):
+        if binding.error:
+            # python-dotenv counts the line from before the blank lines that
+            # come ahead of the statement.
+            original = binding.original.string
+            blank = original[: len(original) - len(original.lstrip())]
+            line = binding.original.line + blank.count("\n")
+            raise OmertaError(f"{name}, line {line}: python-dotenv cannot parse this statement")
+        if binding.key is not None:
+            bindings.append((binding.key, binding.value))
+    return bindings
+
+
+def _decode(data: bytes) -> str | None:
+    """The text of an env file's bytes, or ``None`` where they do not decode."""
+    # The caller raises its own error after this returns, outside of the
+    # handler: the decoding error, which holds the bytes in question, is then
+    # not its context.
+    try:
+        if data.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
+            return data.decode("utf-16")
+        return data.removeprefix(codecs.BOM_UTF8).decode("utf-8")
+    except UnicodeDecodeError:
+        return None
+
+
+def _walk(top: Path, folder: tuple[str, ...], rest: tuple[str, ...]) -> Iterator[tuple[str, ...]]:
+    """The files under ``folder`` whose paths match the remaining pattern parts."""
+    head, tail = rest[0], rest[1:]
+    entries = _entries(top, folder)
+    if head == "**":
+        if tail:
+            yield from _walk(top, folder, tail)
+        else:
+            yield from ((*folder, name) for name, is_folder in entries if not is_folder)
+        for name, is_folder in entries:
+            if is_folder:
+                yield from _walk(top, (*folder, name), rest)
+        return
+    for name, is_folder in entries:
+        if not fnmatch.fnmatchcase(name, head):
+            continue
+        if tail and is_folder:
+            yield from _walk(top, (*folder, name), tail)
+        elif not tail and not is_folder:
+            yield (*folder, name)
+
+
+def _entries(top: Path, folder: tuple[str, ...]) -> list[tuple[str, bool]]:
+    """The folders and files of a folder: each name, and whether it is a folder.
+
+    Left out are ``.git``, a folder holding a ``.git`` of its own, a symbolic
+    link to a folder, and what is neither a folder nor a file.
+    """
+    path = top.joinpath(*folder)
+    entries = []
+    try:
+        with os.scandir(path) as listing:
+            for entry in listing:
+                if entry.name == ".git":
+                    continue
+                if entry.is_dir(follow_symlinks=False):
+                    if not os.path.lexists(os.path.join(entry.path, ".git")):
+                        entries.append((entry.name, True))
+                elif entry.is_file():
+                    entries.append((entry.name, False))
+    except OSError as error:
+        name = quote(os.fsencode("/".join(folder) or "."))
+        raise OmertaError(f"folder {name} cannot be listed ({error.strerror})") from None
+    return entries

omerta/errors.py

@@ -0,0 +1,11 @@
+"""The error that ends a run."""
+
+from __future__ import annotations
+
+
+class OmertaError(Exception):
+    """A failure that ends a run with exit code 2.
+
+    Its message says what failed and where (a file, a line, a git command),
+    never a value of an env file.
+    """