dotguard-cli 0.2.0__py3-none-any.whl

dotguard/parser.py

@@ -0,0 +1,216 @@
+"""A small, dependency-free parser for .env files.
+
+Supported syntax:
+
+    # a comment
+    KEY=value
+    export KEY=value
+    KEY="double quoted with \\n escapes"   # trailing comment
+    KEY='single quoted, taken literally'
+    KEY=value # trailing comment (needs a space before #)
+    KEY="a quoted value
+    can span several lines"
+    URL=http://${HOST}:${PORT:-8080}/   # references to earlier keys
+
+Annotation comments directly above a key attach metadata to it:
+
+    # @type port
+    # @required
+    PORT=8080
+"""
+
+from __future__ import annotations
+
+import re
+
+from dotguard.models import Entry, EnvFile
+
+
+class ParseError(ValueError):
+    def __init__(self, message: str, path: str, line: int) -> None:
+        super().__init__(f"{path}:{line}: {message}")
+        self.path = path
+        self.line = line
+
+
+_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]*$")
+_ANNOTATION_RE = re.compile(r"^@([a-z][a-z0-9_-]*)(?:\s+(.*))?$")
+_ESCAPES = {"n": "\n", "t": "\t", "r": "\r", '"': '"', "\\": "\\", "$": "$"}
+# ${NAME}, ${NAME:-default}, ${NAME-default} or $NAME
+_REFERENCE_RE = re.compile(
+    r"\$(?:\{(?P<braced>[A-Za-z_][A-Za-z0-9_]*)(?:(?P<op>:?-)(?P<default>[^}]*))?\}"
+    r"|(?P<bare>[A-Za-z_][A-Za-z0-9_]*))"
+)
+
+
+def parse_file(path: str, interpolate: bool = True) -> EnvFile:
+    with open(path, encoding="utf-8") as handle:
+        return parse(handle.read(), path=path, interpolate=interpolate)
+
+
+def parse(text: str, path: str = "<string>", interpolate: bool = True) -> EnvFile:
+    """Parse ``text`` into an :class:`EnvFile`.
+
+    With ``interpolate``, ``$NAME`` and ``${NAME}`` in unquoted and
+    double-quoted values are replaced by the value of a key defined earlier in
+    the same file. Names that are not defined expand to an empty string and are
+    recorded in ``Entry.unresolved``.
+    """
+    env = EnvFile(path=path)
+    annotations: dict[str, str] = {}
+    doc: list[str] = []
+    lines = text.splitlines()
+    index = 0
+
+    while index < len(lines):
+        number = index + 1
+        line = lines[index].strip()
+        index += 1
+
+        if not line:
+            # A blank line detaches comments from the next key.
+            annotations, doc = {}, []
+            continue
+
+        if line.startswith("#"):
+            comment = line[1:].strip()
+            match = _ANNOTATION_RE.match(comment)
+            if match:
+                annotations[match.group(1)] = (match.group(2) or "").strip()
+            else:
+                doc.append(comment)
+            continue
+
+        key, rest = _split_assignment(line, path, number)
+
+        # A quoted value may continue on the following lines until its
+        # closing quote.
+        if rest[:1] in ('"', "'"):
+            while _find_closing_quote(rest, rest[0]) is None and index < len(lines):
+                rest += "\n" + lines[index]
+                index += 1
+
+        known = {entry.key: entry.value for entry in env.entries} if interpolate else None
+        unresolved: list[str] = []
+        value = _parse_value(rest, path, number, known, unresolved)
+
+        if env.get(key) is not None:
+            raise ParseError(f"duplicate key {key!r}", path, number)
+        env.entries.append(
+            Entry(
+                key=key,
+                value=value,
+                line=number,
+                annotations=annotations,
+                doc=doc,
+                unresolved=unresolved,
+            )
+        )
+        annotations, doc = {}, []
+
+    return env
+
+
+def _split_assignment(line: str, path: str, number: int) -> tuple[str, str]:
+    if line.startswith("export "):
+        line = line[len("export "):].lstrip()
+
+    if "=" not in line:
+        raise ParseError("expected KEY=value", path, number)
+
+    key, rest = line.split("=", 1)
+    key = key.strip()
+    if not _KEY_RE.match(key):
+        raise ParseError(f"invalid key {key!r}", path, number)
+    return key, rest.strip()
+
+
+def _parse_value(
+    rest: str,
+    path: str,
+    number: int,
+    known: dict[str, str] | None,
+    unresolved: list[str],
+) -> str:
+    if rest.startswith("'"):
+        # Single quotes: everything is literal, no escapes, no references.
+        end = _find_closing_quote(rest, "'")
+        if end is None:
+            raise ParseError("unterminated quoted value", path, number)
+        value, tail = rest[1:end], rest[end + 1:]
+    elif rest.startswith('"'):
+        end = _find_closing_quote(rest, '"')
+        if end is None:
+            raise ParseError("unterminated quoted value", path, number)
+        value = _unescape(rest[1:end], known, unresolved)
+        tail = rest[end + 1:]
+    else:
+        value = _unescape(_strip_inline_comment(rest), known, unresolved, escapes={"$": "$"})
+        tail = ""
+
+    if tail and not tail.lstrip().startswith("#"):
+        raise ParseError("unexpected text after closing quote", path, number)
+    return value
+
+
+def _find_closing_quote(text: str, quote: str) -> int | None:
+    i = 1
+    while i < len(text):
+        char = text[i]
+        if char == "\\" and quote == '"':
+            i += 2
+            continue
+        if char == quote:
+            return i
+        i += 1
+    return None
+
+
+def _unescape(
+    value: str,
+    known: dict[str, str] | None,
+    unresolved: list[str],
+    escapes: dict[str, str] = _ESCAPES,
+) -> str:
+    """Apply backslash escapes and, when ``known`` is given, expand references."""
+    out = []
+    i = 0
+    while i < len(value):
+        char = value[i]
+        if char == "\\" and i + 1 < len(value) and value[i + 1] in escapes:
+            out.append(escapes[value[i + 1]])
+            i += 2
+            continue
+        if char == "$" and known is not None:
+            match = _REFERENCE_RE.match(value, i)
+            if match:
+                out.append(_resolve(match, known, unresolved))
+                i = match.end()
+                continue
+        out.append(char)
+        i += 1
+    return "".join(out)
+
+
+def _resolve(match: re.Match[str], known: dict[str, str], unresolved: list[str]) -> str:
+    name = match.group("braced") or match.group("bare")
+    op, default = match.group("op"), match.group("default")
+    value = known.get(name)
+
+    if op == ":-" and not value:
+        return default
+    if op == "-" and value is None:
+        return default
+    if value is None:
+        if name not in unresolved:
+            unresolved.append(name)
+        return ""
+    return value
+
+
+def _strip_inline_comment(value: str) -> str:
+    # "a#b" stays as-is (common in URLs); only " #" starts a comment.
+    match = re.search(r"\s#", value)
+    if match:
+        value = value[: match.start()]
+    return value.strip()

dotguard/report.py

@@ -0,0 +1,79 @@
+"""Render issues for people (text) and for tools (JSON, GitHub annotations)."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from typing import TextIO
+
+from dotguard.models import Issue, Severity
+
+FORMATS = ("text", "json", "github")
+
+
+def _use_color(stream: TextIO) -> bool:
+    if os.environ.get("NO_COLOR"):
+        return False
+    if os.environ.get("FORCE_COLOR"):
+        return True
+    return hasattr(stream, "isatty") and stream.isatty()
+
+
+def _check_mark(stream: TextIO) -> str:
+    # Windows consoles often use a legacy code page that cannot encode "✓".
+    try:
+        "✓".encode(getattr(stream, "encoding", None) or "utf-8")
+    except (UnicodeEncodeError, LookupError):
+        return "OK:"
+    return "✓"
+
+
+def _paint(text: str, code: str, enabled: bool) -> str:
+    return f"\033[{code}m{text}\033[0m" if enabled else text
+
+
+def render(issues: list[Issue], fmt: str = "text", stream: TextIO | None = None) -> None:
+    # Resolve stdout at call time so redirection (and tests) work.
+    stream = stream or sys.stdout
+    if fmt == "json":
+        json.dump({"issues": [i.to_dict() for i in issues], "summary": summary(issues)}, stream, indent=2)
+        stream.write("\n")
+    elif fmt == "github":
+        _render_github(issues, stream)
+    else:
+        _render_text(issues, stream)
+
+
+def summary(issues: list[Issue]) -> dict:
+    errors = sum(1 for i in issues if i.severity is Severity.ERROR)
+    return {"errors": errors, "warnings": len(issues) - errors}
+
+
+def _render_text(issues: list[Issue], stream: TextIO) -> None:
+    color = _use_color(stream)
+    for item in issues:
+        location = item.path if item.line is None else f"{item.path}:{item.line}"
+        if item.severity is Severity.ERROR:
+            label = _paint("error", "1;31", color)
+        else:
+            label = _paint("warning", "1;33", color)
+        stream.write(f"{_paint(location, '2', color)}  {label} {item.code}  {item.message}\n")
+
+    counts = summary(issues)
+    if not issues:
+        stream.write(_paint(f"{_check_mark(stream)} no problems found", "32", color) + "\n")
+    else:
+        stream.write(f"\n{counts['errors']} error(s), {counts['warnings']} warning(s)\n")
+
+
+def _render_github(issues: list[Issue], stream: TextIO) -> None:
+    # https://docs.github.com/actions/reference/workflow-commands-for-github-actions
+    for item in issues:
+        level = "error" if item.severity is Severity.ERROR else "warning"
+        props = f"file={item.path}"
+        if item.line is not None:
+            props += f",line={item.line}"
+        props += f",title=dotguard {item.code}"
+        message = item.message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
+        stream.write(f"::{level} {props}::{message}\n")

dotguard/secrets.py

@@ -0,0 +1,90 @@
+"""Find real-looking secrets in files that are meant to be committed."""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+
+from dotguard.models import EnvFile, Issue, Severity
+
+# Well-known credential formats. Each match is an error: these are never
+# placeholders.
+KNOWN_TOKENS: list[tuple[str, re.Pattern[str]]] = [
+    ("AWS access key ID", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")),
+    ("GitHub token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,}\b")),
+    ("GitHub fine-grained token", re.compile(r"\bgithub_pat_[A-Za-z0-9_]{60,}\b")),
+    ("Slack token", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}\b")),
+    ("Stripe live key", re.compile(r"\b[sr]k_live_[A-Za-z0-9]{16,}\b")),
+    ("Google API key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b")),
+    ("OpenAI-style API key", re.compile(r"\bsk-(?:proj-)?[A-Za-z0-9_-]{32,}\b")),
+    ("private key", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")),
+    ("JSON Web Token", re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}")),
+]
+
+SENSITIVE_NAME = re.compile(
+    r"(SECRET|TOKEN|PASSWORD|PASSWD|PWD|PRIVATE|CREDENTIAL|API_?KEY|ACCESS_?KEY|AUTH)",
+    re.IGNORECASE,
+)
+
+# Values that are obviously documentation, not real credentials.
+PLACEHOLDER = re.compile(
+    r"^(?:|changeme|change[-_]me|replace[-_]?me|todo|secret|password|example.*|your[-_].*|"
+    r"<.*>|\{\{.*\}\}|\$\{.*\}|x+|\*+|\.\.\.|null|none|test|dummy.*|fake.*|placeholder.*)$",
+    re.IGNORECASE,
+)
+
+ENTROPY_THRESHOLD = 3.5
+MIN_SECRET_LENGTH = 16
+
+
+def shannon_entropy(value: str) -> float:
+    if not value:
+        return 0.0
+    counts = Counter(value)
+    total = len(value)
+    return -sum((n / total) * math.log2(n / total) for n in counts.values())
+
+
+def scan(env: EnvFile) -> list[Issue]:
+    """Report values in ``env`` that look like real credentials."""
+    issues: list[Issue] = []
+
+    for entry in env.entries:
+        value = entry.value
+
+        known = next((name for name, regex in KNOWN_TOKENS if regex.search(value)), None)
+        if known:
+            issues.append(
+                Issue(
+                    "DS001",
+                    f"{entry.key} contains what looks like a real {known}",
+                    Severity.ERROR,
+                    env.path,
+                    entry.line,
+                    entry.key,
+                )
+            )
+            continue
+
+        if PLACEHOLDER.match(value):
+            continue
+
+        looks_sensitive = SENSITIVE_NAME.search(entry.key) or "secret" in entry.annotations
+        if (
+            looks_sensitive
+            and len(value) >= MIN_SECRET_LENGTH
+            and shannon_entropy(value) >= ENTROPY_THRESHOLD
+        ):
+            issues.append(
+                Issue(
+                    "DS002",
+                    f"{entry.key} has a high-entropy value; use a placeholder in shared files",
+                    Severity.WARNING,
+                    env.path,
+                    entry.line,
+                    entry.key,
+                )
+            )
+
+    return issues

dotguard/types.py

@@ -0,0 +1,76 @@
+"""Value validators used by the ``@type`` annotation."""
+
+from __future__ import annotations
+
+import re
+from typing import Callable, Optional
+from urllib.parse import urlparse
+
+# A validator returns None when the value is valid, or a short reason otherwise.
+Validator = Callable[[str], Optional[str]]
+
+_TRUE = {"1", "true", "yes", "on"}
+_FALSE = {"0", "false", "no", "off"}
+_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
+
+
+def _string(value: str) -> str | None:
+    return None
+
+
+def _int(value: str) -> str | None:
+    try:
+        int(value, 10)
+    except ValueError:
+        return "expected an integer"
+    return None
+
+
+def _float(value: str) -> str | None:
+    try:
+        float(value)
+    except ValueError:
+        return "expected a number"
+    return None
+
+
+def _bool(value: str) -> str | None:
+    if value.lower() in _TRUE | _FALSE:
+        return None
+    return "expected a boolean (true/false, yes/no, on/off, 1/0)"
+
+
+def _port(value: str) -> str | None:
+    if _int(value) is not None or not 1 <= int(value) <= 65535:
+        return "expected a port number between 1 and 65535"
+    return None
+
+
+def _url(value: str) -> str | None:
+    parsed = urlparse(value)
+    if parsed.scheme and parsed.netloc:
+        return None
+    return "expected a URL with a scheme and host, e.g. https://example.com"
+
+
+def _email(value: str) -> str | None:
+    if _EMAIL_RE.match(value):
+        return None
+    return "expected an email address"
+
+
+VALIDATORS: dict[str, Validator] = {
+    "string": _string,
+    "str": _string,
+    "int": _int,
+    "integer": _int,
+    "float": _float,
+    "number": _float,
+    "bool": _bool,
+    "boolean": _bool,
+    "port": _port,
+    "url": _url,
+    "email": _email,
+}
+
+NUMERIC_TYPES = {"int", "integer", "float", "number", "port"}