scratchrelaxtv 0.7.0__py3-none-any.whl

scratchrelaxtv/__init__.py

@@ -0,0 +1,552 @@
+"""scratchrelaxtv module.
+
+scratchrelaxtv (anagram of extract-hcl-vars) is a Terraform module development
+tool. Use it to:
+
+* extract vars from HCL and generate an HCL variables file;
+* generate a module-use stub file;
+* delete files generated by scratchrelaxtv; and
+* check whether a module declares every variable it uses and uses every
+  variable it declares.
+
+By default, Terraform files are read from every ``*.tf`` file in the current
+directory, so modules split across several files are handled as a whole.
+
+Example:
+    Help using the scratchrelaxtv CLI can be found by typing the following::
+
+        $ scratchrelaxtv --help
+"""
+
+from __future__ import annotations
+
+import argparse
+import functools
+import glob
+import logging
+import os
+import re
+
+__version__ = "0.7.0"
+EXIT_OKAY = 0
+EXIT_NOT_OKAY = 1
+
+logger = logging.getLogger(__name__)
+
+# HCL identifiers are Unicode: a letter or underscore, then letters, digits, "_", or "-".
+_HEREDOC_START = re.compile(r"<<-?([^\W\d][\w-]*)[ \t]*\r?\n")
+# Runs of characters that can't start a comment, string, heredoc, or brace.
+_PLAIN_EXPR = re.compile(r'[^#/"<{}]+')
+_PLAIN_QUOTED = re.compile(r'[^\\"\n$%]+')
+_PLAIN_HEREDOC = re.compile(r"[^\n$%]+")
+
+
+def remove_prefix(text: str, prefix: str) -> str:
+    """Remove prefix from a string."""
+    return text[text.startswith(prefix) and len(prefix) :]
+
+
+def aligned(pairs: list[tuple[str, str]], indent: str = "") -> str:
+    """Return ``name = value`` lines with ``=`` aligned the way ``terraform fmt`` does."""
+    width = max((len(name) for name, _ in pairs), default=0)
+    return "".join(f"{indent}{name.ljust(width)} = {value}\n" for name, value in pairs)
+
+
+def variable_block(name: str) -> str:
+    """Return a variable declaration with no default, so the input stays required."""
+    body = aligned([("description", '""'), ("type", "any")], indent="  ")
+    return f'variable "{name}" {{\n{body}}}\n'
+
+
+def _blank(text: str) -> str:
+    """Replace every character except line breaks with a space."""
+    return "\n".join(" " * len(line) for line in text.split("\n"))
+
+
+class _Scanner:
+    """Split HCL into comment-free code, its expressions, and its top-level parts.
+
+    Tracks quoted strings, heredocs, ``${...}``/``%{...}`` template sequences (which
+    may contain nested strings), and brace depth, so comment markers and
+    declaration-like text are only recognized where HCL treats them as such.
+    """
+
+    def __init__(self, text: str) -> None:
+        self.text = text
+        self.i = 0
+        self.code: list[str] = []
+        self.top: list[str] = []
+        self.expr: list[str] = []
+
+    def emit(self, text: str, top: bool, literal: bool = False) -> None:
+        """Add text; ``literal`` marks string or heredoc text that isn't an expression."""
+        self.code.append(text)
+        self.top.append(text if top else _blank(text))
+        self.expr.append(_blank(text) if literal else text)
+
+    def startswith(self, prefix: str) -> bool:
+        return self.text.startswith(prefix, self.i)
+
+    def expression(self, top: bool, in_template: bool) -> None:
+        """Scan expressions until EOF or, in a template sequence, its closing brace."""
+        text, depth = self.text, 0
+        while self.i < len(text):
+            char, here = text[self.i], top and depth == 0
+            if plain := _PLAIN_EXPR.match(text, self.i):
+                self.emit(plain[0], here)
+                self.i = plain.end()
+            elif char == "#" or self.startswith("//"):
+                newline = text.find("\n", self.i)
+                self.emit(" ", here)
+                self.i = len(text) if newline < 0 else newline
+            elif self.startswith("/*"):
+                close = text.find("*/", self.i + 2)
+                stop = len(text) if close < 0 else close + 2
+                self.emit(" " + "\n" * text.count("\n", self.i, stop), here)
+                self.i = stop
+            elif char == '"':
+                self.quoted(here)
+            elif self.startswith("<<") and (heredoc := _HEREDOC_START.match(text, self.i)):
+                self.heredoc(heredoc, here)
+            elif char == "{":
+                self.emit(char, here)
+                depth += 1
+                self.i += 1
+            elif char == "}":
+                self.i += 1
+                if depth == 0 and in_template:
+                    self.emit(char, False)
+                    return
+                depth = max(depth - 1, 0)
+                self.emit(char, top and depth == 0)
+            else:
+                self.emit(char, here)
+                self.i += 1
+
+    def template_part(self, top: bool) -> bool:
+        """Handle an escape or template sequence at the cursor; return True if one was found."""
+        if self.startswith("$${") or self.startswith("%%{"):
+            self.emit(self.text[self.i : self.i + 3], top, literal=True)
+            self.i += 3
+        elif self.startswith("${") or self.startswith("%{"):
+            self.emit(self.text[self.i : self.i + 2], top, literal=True)
+            self.i += 2
+            self.expression(top=False, in_template=True)
+        else:
+            return False
+        return True
+
+    def quoted(self, top: bool) -> None:
+        text = self.text
+        self.emit('"', top, literal=True)
+        self.i += 1
+        while self.i < len(text):
+            char = text[self.i]
+            if char == "\\":
+                self.emit(text[self.i : self.i + 2], top, literal=True)
+                self.i += 2
+            elif plain := _PLAIN_QUOTED.match(text, self.i):
+                self.emit(plain[0], top, literal=True)
+                self.i = plain.end()
+            elif char in '"\n':
+                if char == '"':
+                    self.emit(char, top, literal=True)
+                    self.i += 1
+                return
+            elif not self.template_part(top):
+                self.emit(char, top, literal=True)
+                self.i += 1
+
+    def heredoc(self, start: re.Match[str], top: bool) -> None:
+        text = self.text
+        closing = re.compile(rf"[ \t]*{re.escape(start[1])}[ \t]*(?=\r?\n|$)")
+        self.emit(start[0], top, literal=True)
+        self.i = start.end()
+        line_start = True
+        while self.i < len(text):
+            if line_start and (end := closing.match(text, self.i)):
+                self.emit(end[0], top, literal=True)
+                self.i = end.end()
+                return
+            char = text[self.i]
+            line_start = False
+            if plain := _PLAIN_HEREDOC.match(text, self.i):
+                self.emit(plain[0], top, literal=True)
+                self.i = plain.end()
+            elif not self.template_part(top):
+                self.emit(char, top, literal=True)
+                self.i += 1
+                line_start = char == "\n"
+
+
+@functools.lru_cache(maxsize=256)
+def _scan(text: str) -> tuple[str, str, str]:
+    """Return (code without comments, top-level parts, expressions) for HCL text."""
+    scanner = _Scanner(text)
+    scanner.expression(top=True, in_template=False)
+    return "".join(scanner.code), "".join(scanner.top), "".join(scanner.expr)
+
+
+def strip_comments(text: str) -> str:
+    """Replace HCL comments (``#``, ``//``, ``/* */``) with spaces, keeping line breaks.
+
+    Comment markers inside quoted strings (e.g., ``"https://..."``), template
+    sequences (``"${format("https://%s", var.host)}"``), and heredocs (e.g., a
+    ``#!/bin/bash`` line in ``user_data``) are not comments and are kept.
+    """
+    return _scan(text)[0]
+
+
+def top_level(text: str) -> str:
+    """Return only the unnested, uncommented parts of HCL; everything else becomes spaces.
+
+    Block headers such as ``variable "a" {`` are kept, while text in block bodies,
+    strings used as values, and heredocs is blanked, so it can't look like a
+    declaration.
+    """
+    return _scan(text)[1]
+
+
+def expressions(text: str) -> str:
+    """Return HCL with comments and the literal text of strings and heredocs blanked.
+
+    Expressions, including those inside ``${...}`` and ``%{...}`` template
+    sequences, are kept, so ``"Replaces var.old"`` is not a reference but
+    ``"${var.new}"`` is.
+    """
+    return _scan(text)[2]
+
+
+# Directories -r never descends into: VCS data and downloaded Terraform modules.
+_SKIP_DIRS = frozenset({".git", ".terraform", ".terragrunt-cache"})
+
+
+def uses(text: str) -> str:
+    """Return :func:`expressions`, minus each variable's references to itself.
+
+    A ``var.x`` inside ``variable "x" { validation { ... } }`` checks the value but
+    doesn't use it, so it shouldn't hide an unused ``x``. References to other
+    variables in a validation block still count.
+    """
+    _, top, expr = _scan(text)
+    for header in re.finditer(r'\bvariable\s+"([^"]+)"\s*\{', top):
+        close = top.find("}", header.end())
+        end = len(top) if close < 0 else close
+        own = re.compile(rf"\bvar\.{re.escape(header[1])}(?![\w-])")
+        body = own.sub(lambda m: " " * len(m[0]), expr[header.end() : end])
+        expr = expr[: header.end()] + body + expr[end:]
+    return expr
+
+
+def remove_files() -> None:
+    """Remove files from the os that look like scratchrelaxtv files."""
+    logger.info("attempting removal of files")
+    pattern = r"^(variables\.\d+|modstub(\.\d+|))\.tf$"
+    for root, dirs, files in os.walk(os.getcwd()):
+        dirs[:] = [d for d in dirs if d not in _SKIP_DIRS]
+        for file in filter(lambda x: re.match(pattern, x), files):
+            os.remove(os.path.join(root, file))
+            logger.info("removed file: %s", os.path.join(root, file))
+
+
+class BassExtractor:
+    """Extract variables from Terraform HCL files to ease module creation."""
+
+    # valid var names:
+    # "A name must start with a letter and may contain only letters,
+    # digits, underscores, and dashes."
+    var_regex = r"\bvar\.([a-zA-Z][a-zA-Z0-9_-]*)\b"
+    variable_regex = r'\bvariable\s+"([^"]+)"'
+
+    # Inputs are HCL, so comments are ignored. Subclasses reading other
+    # formats (e.g., shell templates) turn this off.
+    hcl_input = True
+
+    def __init__(self, args: argparse.Namespace) -> None:
+        """Store parsed args and log the attributes of this run."""
+        self.args = args
+        self.tf_vars: list[str] = []
+        self.log_arguments()
+
+    @property
+    def input_files(self) -> list[str]:
+        """Return the input file, or every ``*.tf`` file in the current directory."""
+        if self.args.input:
+            return [self.args.input]
+        return sorted(glob.glob("*.tf"))
+
+    def log_arguments(self) -> None:
+        """Log the attributes of this run."""
+        logger.info("input files: %s", ", ".join(self.input_files) or "(none)")
+        logger.info("output file: %s", self.args.output)
+
+        if self.args.force:
+            logger.info("forcing overwrite of output file")
+        else:
+            logger.info("not forcing overwrite of output file")
+
+        if self.args.asc:
+            logger.info("ordering output file ascending")
+        elif self.args.desc:
+            logger.info("ordering output file descending")
+        else:
+            logger.info("not ordering output file")
+
+    def _find_non_existing_filename(self) -> None:
+        index = 1
+        while True:
+            if self.args.force or not os.path.isfile(self.args.output):
+                return
+            filename, file_extension = os.path.splitext(self.args.output)
+            pattern = re.compile(r"(.*)\.(\d+)")
+            search = pattern.search(filename)
+            if search:
+                filename = search.group(1)
+                index = int(search.group(2)) + 1
+            self.args.output = "".join([filename, ".", str(index), file_extension])
+            index += 1
+
+    @staticmethod
+    def get_file_contents(filename: str) -> str:
+        """Return contents of file as a string."""
+        with open(filename, encoding="utf-8") as file_handle:
+            return file_handle.read()
+
+    @staticmethod
+    def find_vars(haystack: str, regex: str) -> list[str]:
+        """Return unique regex matches in order of first appearance."""
+        var_pattern = re.compile(regex)
+        tf_vars: list[str] = var_pattern.findall(haystack)
+        return list(dict.fromkeys(tf_vars))
+
+    def read(self, filenames: list[str], top: bool = False) -> str:
+        """Return the contents of files, reduced to the parts that matter if HCL.
+
+        With ``top``, return only the top level of each HCL file (see :func:`top_level`);
+        otherwise, only its expressions (see :func:`expressions`).
+        """
+        texts = [self.get_file_contents(name) for name in filenames]
+        if self.hcl_input:
+            texts = [top_level(t) if top else expressions(t) for t in texts]
+        return "\n".join(texts)
+
+    def find_vars_in_file(self, regex: str, top: bool = False) -> None:
+        """Extract vars from the input files."""
+        self.tf_vars = self.find_vars(self.read(self.input_files, top), regex)
+
+        if self.args.asc:
+            self.tf_vars.sort()
+        elif self.args.desc:
+            self.tf_vars.sort(reverse=True)
+
+
+class VarExtractor(BassExtractor):
+    """Extract variables from Terraform HCL files to ease module creation."""
+
+    def __init__(self, args: argparse.Namespace) -> None:
+        """Set output default, then initialize the extractor."""
+        logger.info("generating variables file")
+
+        if not args.output:
+            args.output = "variables.tf"
+
+        super().__init__(args)
+
+    def write_file(self) -> None:
+        """Output vars to .tf file."""
+        self._find_non_existing_filename()
+        with open(self.args.output, "w", encoding="utf-8") as file_handle:
+            file_handle.write(
+                "\n".join(variable_block(remove_prefix(v, "var.")) for v in self.tf_vars)
+            )
+
+    def extract(self) -> int:
+        """Extract vars from .tf files."""
+        self.find_vars_in_file(self.var_regex)
+        self.write_file()
+
+        return EXIT_OKAY
+
+
+class StubMaker(BassExtractor):
+    """Extract variables from Terraform HCL files and generate a module stub."""
+
+    def __init__(self, args: argparse.Namespace) -> None:
+        """Set stub output/module-name defaults, then initialize."""
+        logger.info("generating module usage stub")
+
+        if not args.output:
+            args.output = "modstub.tf"
+
+        if not args.modname:
+            args.modname = os.path.basename(os.getcwd())
+
+        super().__init__(args)
+
+    def write_file(self) -> None:
+        """Output vars to .tf file."""
+        self._find_non_existing_filename()
+        # No providers block: a child module inherits the parent's default
+        # provider configurations, and hardcoding one assumed AWS.
+        modname = self.args.modname
+        args = aligned([(v, f"local.{v}") for v in self.tf_vars], indent="  ")
+        with open(self.args.output, "w", encoding="utf-8") as file_handle:
+            file_handle.write(f'module "{modname}" {{\n  source = "../{modname}"\n')
+            if args:
+                file_handle.write(f"\n{args}")
+            file_handle.write("}\n")
+
+    def extract(self) -> int:
+        """Extract declared variables from .tf files."""
+        self.find_vars_in_file(self.variable_regex, top=True)
+        self.write_file()
+
+        return EXIT_OKAY
+
+
+class EnvGenerator(BassExtractor):
+    """Extract variables from Terraform HCL files and generate a .env file."""
+
+    def __init__(self, args: argparse.Namespace) -> None:
+        """Set .env/.tfvars output defaults, then initialize."""
+        if args.env:
+            logger.info("generating .env file")
+        elif args.tfvars:
+            logger.info("generating .tfvars file")
+
+        if not args.output and args.env:
+            args.output = ".env"
+
+        if not args.output and args.tfvars:
+            args.output = "terraform.tfvars"
+
+        super().__init__(args)
+
+    def _write_env(self) -> None:
+        """Output vars to .env file."""
+        with open(self.args.output, "w", encoding="utf-8") as file_handle:
+            file_handle.write('unset "${!TF_VAR_@}"\n')
+            for tf_var in self.tf_vars:
+                file_handle.write("".join(["export TF_VAR_", tf_var, "=replace\n"]))
+
+    def _write_tfvars(self) -> None:
+        """Output vars to .tfvars file."""
+        with open(self.args.output, "w", encoding="utf-8") as file_handle:
+            file_handle.write(aligned([(v, '"replace"') for v in self.tf_vars]))
+
+    def write_file(self) -> None:
+        """Output vars to file."""
+        self._find_non_existing_filename()
+        if self.args.env:
+            self._write_env()
+        elif self.args.tfvars:
+            self._write_tfvars()
+
+    def extract(self) -> int:
+        """Extract vars from .tf files."""
+        self.find_vars_in_file(self.var_regex)
+        self.write_file()
+
+        return EXIT_OKAY
+
+
+class Checker(BassExtractor):
+    """Check that a module declares the variables it uses, and uses the ones it declares."""
+
+    def __init__(self, args: argparse.Namespace, label: str = ".") -> None:
+        """Set check output default, then initialize.
+
+        ``label`` names the module directory in warnings.
+        """
+        logger.info("checking for undeclared and unused variables")
+        self.label = label
+
+        if not args.output:
+            args.output = "variables.tf"
+
+        super().__init__(args)
+
+    def write_file(self) -> None:
+        """Append declarations for ``self.tf_vars`` to the output file."""
+        existing = (
+            self.get_file_contents(self.args.output) if os.path.isfile(self.args.output) else ""
+        )
+        if not existing.strip():
+            separator = ""
+        elif existing.endswith("\n"):
+            separator = "\n"
+        else:
+            separator = "\n\n"
+        blocks = "\n".join(variable_block(v) for v in self.tf_vars)
+        with open(self.args.output, "a", encoding="utf-8") as file_handle:
+            file_handle.write(separator + blocks)
+
+    def find_missing(self) -> dict[str, list[str]]:
+        """Return undeclared (used, not declared) and unused (declared, not used) vars."""
+        text = "\n".join(uses(self.get_file_contents(name)) for name in self.input_files)
+        used = self.find_vars(text, self.var_regex)
+
+        declaration_files = list(self.input_files)
+        if os.path.isfile(self.args.output) and self.args.output not in declaration_files:
+            declaration_files.append(self.args.output)
+        declared = self.find_vars(self.read(declaration_files, top=True), self.variable_regex)
+
+        return {
+            "undeclared": [v for v in used if v not in declared],
+            "unused": [v for v in declared if v not in used],
+        }
+
+    def extract(self) -> int:
+        """Report problems, add undeclared variables with ``-f``, and return the exit code."""
+        missing = self.find_missing()
+
+        if missing["undeclared"]:
+            logger.warning(
+                "%s: undeclared variables: %s", self.label, ", ".join(missing["undeclared"])
+            )
+            if self.args.force:
+                self.tf_vars = missing["undeclared"]
+                self.write_file()
+                logger.warning("%s: added declarations to %s", self.label, self.args.output)
+        if missing["unused"]:
+            logger.warning("%s: unused variables: %s", self.label, ", ".join(missing["unused"]))
+
+        if missing["undeclared"] or missing["unused"]:
+            return EXIT_NOT_OKAY
+        return EXIT_OKAY
+
+
+class TemplateExtractor(BassExtractor):
+    """Extract template variables from Terraform templates and emit HCL."""
+
+    var_regex = r"(?<!\$)\${([a-zA-Z][a-zA-Z0-9_-]+)}"
+    hcl_input = False
+
+    def __init__(self, args: argparse.Namespace) -> None:
+        """Set template input/output defaults, then initialize."""
+        logger.info("extracting template variables")
+
+        if not args.input:
+            args.input = "template.sh"
+
+        if not args.output:
+            args.output = "template_vars.tf"
+
+        super().__init__(args)
+
+    def write_file(self) -> None:
+        """Output vars to .tf file."""
+        self._find_non_existing_filename()
+
+        with open(self.args.output, "w", encoding="utf-8") as file_handle:
+            file_handle.write("locals {\n")
+            file_handle.write("  templates_vars = {\n")
+            file_handle.write(aligned([(v, '"replace"') for v in self.tf_vars], indent="    "))
+            file_handle.write("  }\n}\n")
+
+    def extract(self) -> int:
+        """Extract vars from the template file."""
+        self.find_vars_in_file(self.var_regex)
+        self.write_file()
+
+        return EXIT_OKAY

scratchrelaxtv/__main__.py

@@ -0,0 +1,7 @@
+"""Allow ``python -m scratchrelaxtv``."""
+
+import sys
+
+from scratchrelaxtv.cli import main
+
+sys.exit(main())